Skip to content

Commit 440e58b

Browse files
mattrobenoltmattrobenolt
committed
Pre-compile yaml file into py files
This is a tremendous speedup vs parsing JSON Fixes ua-parser#23
1 parent aae1341 commit 440e58b

File tree

6 files changed

+198
-152
lines changed

6 files changed

+198
-152
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
*.pyc
22
*.egg-info/
33
.eggs/
4+
.cache/
45
build/
56
dist/
67
tmp/
78
regexes.yaml
8-
regexes.json
9+
_regexes.py

‎MANIFEST.in‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1 @@
11
include README.md
2-
include ua_parser/regexes.json

‎Makefile‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,18 @@
1-
all: prep test
1+
all: test
22

3-
prep:
4-
#git submodule update --init
5-
#sudo apt-get install python-yaml
6-
7-
test:
8-
@#test ! -d tmp && mkdir tmp
9-
@export PYTHONPATH=tmp && python setup.py develop -d tmp
3+
test: clean
4+
@mkdir -p tmp
5+
@PYTHONPATH=tmp python setup.py develop -d tmp
106
@# run all tests
11-
@python ua_parser/user_agent_parser_test.py
7+
@PYTHONPATH=tmp python ua_parser/user_agent_parser_test.py
128
@# run a single test
13-
@#python ua_parser/user_agent_parser_test.py ParseTest.testStringsDeviceBrandModel
9+
@#PYTHONPATH=tmp python ua_parser/user_agent_parser_test.py ParseTest.testStringsDeviceBrandModel
1410

1511
clean:
16-
@rm -f ua_parser/user_agent_parser.pyc\
17-
ua_parser/regexes.yaml\
18-
ua_parser/regexes.json
12+
@find . -name '*.pyc' -delete
1913
@rm -rf tmp\
2014
ua_parser.egg-info\
2115
dist\
2216
build
2317

24-
.PHONY: all prep test clean
18+
.PHONY: all test clean

‎setup.py‎

Lines changed: 90 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,107 @@
11
#!/usr/bin/env python
2+
import os
3+
from distutils import log
4+
from distutils.core import Command
25
from setuptools import setup
36
from setuptools.command.develop import develop as _develop
47
from setuptools.command.sdist import sdist as _sdist
58

69

7-
def install_regexes():
8-
print('Copying regexes.yaml to package directory...')
9-
import os
10-
cwd = os.path.abspath(os.path.dirname(__file__))
11-
yaml_src = os.path.join(cwd, 'uap-core', 'regexes.yaml')
12-
if not os.path.exists(yaml_src):
13-
raise RuntimeError(
14-
'Unable to find regexes.yaml, should be at %r' % yaml_src)
10+
class build_regexes(Command):
11+
description = 'build supporting regular expressions from uap-core'
12+
user_options = [
13+
('work-path=', 'w',
14+
'The working directory for source files. Defaults to .'),
15+
]
1516

16-
print('Converting regexes.yaml to regexes.json...')
17-
import json
18-
import yaml
19-
json_dest = os.path.join(cwd, 'ua_parser', 'regexes.json')
20-
with open(yaml_src, 'rb') as fp:
21-
regexes = yaml.safe_load(fp)
22-
with open(json_dest, "w") as f:
23-
json.dump(regexes, f)
17+
def initialize_options(self):
18+
self.work_path = None
19+
20+
def finalize_options(self):
21+
if self.work_path is None:
22+
self.work_path = os.path.realpath(os.path.join(os.path.dirname(__file__)))
23+
24+
def run(self):
25+
work_path = self.work_path
26+
if os.path.exists(os.path.join(work_path, '.git')):
27+
from subprocess import check_output
28+
log.info("initializing git submodules")
29+
check_output(['git', 'submodule', 'init'], cwd=work_path)
30+
check_output(['git', 'submodule', 'update'], cwd=work_path)
31+
32+
yaml_src = os.path.join(work_path, 'uap-core', 'regexes.yaml')
33+
if not os.path.exists(yaml_src):
34+
raise RuntimeError(
35+
'Unable to find regexes.yaml, should be at %r' % yaml_src)
36+
37+
log.info('Converting regexes.yaml to _regexes.py...')
38+
import yaml
39+
py_dest = os.path.join(work_path, 'ua_parser', '_regexes.py')
40+
with open(yaml_src, 'rb') as fp:
41+
regexes = yaml.safe_load(fp)
42+
with open(py_dest, 'wb') as fp:
43+
fp.write('############################################\n')
44+
fp.write('# NOTICE: This file is autogenerated from #\n')
45+
fp.write('# regexes.yaml. Do not edit by hand, #\n')
46+
fp.write('# instead, re-run `setup.py build_regexes` #\n')
47+
fp.write('############################################\n')
48+
fp.write('\n')
49+
fp.write('from .user_agent_parser import (\n')
50+
fp.write(' UserAgentParser, DeviceParser, OSParser,\n')
51+
fp.write(')\n')
52+
fp.write('\n')
53+
fp.write('__all__ = (\n')
54+
fp.write(' \'USER_AGENT_PARSERS\', \'DEVICE_PARSERS\', \'OS_PARSERS\',\n')
55+
fp.write(')\n')
56+
fp.write('\n')
57+
fp.write('USER_AGENT_PARSERS = [\n')
58+
for device_parser in regexes['user_agent_parsers']:
59+
fp.write(' UserAgentParser(\n')
60+
fp.write(' %r,\n' % device_parser['regex'])
61+
fp.write(' %r,\n' % device_parser.get('family_replacement'))
62+
fp.write(' %r,\n' % device_parser.get('v1_replacement'))
63+
fp.write(' %r,\n' % device_parser.get('v2_replacement'))
64+
fp.write(' ),\n')
65+
fp.write(']\n')
66+
fp.write('\n')
67+
fp.write('DEVICE_PARSERS = [\n')
68+
for device_parser in regexes['device_parsers']:
69+
fp.write(' DeviceParser(\n')
70+
fp.write(' %r,\n' % device_parser['regex'])
71+
fp.write(' %r,\n' % device_parser.get('regex_flag'))
72+
fp.write(' %r,\n' % device_parser.get('device_replacement'))
73+
fp.write(' %r,\n' % device_parser.get('brand_replacement'))
74+
fp.write(' %r,\n' % device_parser.get('model_replacement'))
75+
fp.write(' ),\n')
76+
fp.write(']\n')
77+
fp.write('\n')
78+
fp.write('OS_PARSERS = [\n')
79+
for device_parser in regexes['os_parsers']:
80+
fp.write(' OSParser(\n')
81+
fp.write(' %r,\n' % device_parser['regex'])
82+
fp.write(' %r,\n' % device_parser.get('os_replacement'))
83+
fp.write(' %r,\n' % device_parser.get('os_v1_replacement'))
84+
fp.write(' %r,\n' % device_parser.get('os_v2_replacement'))
85+
fp.write(' ),\n')
86+
fp.write(']\n')
87+
fp.write('\n')
2488

2589

2690
class develop(_develop):
2791
def run(self):
28-
install_regexes()
92+
self.run_command('build_regexes')
2993
_develop.run(self)
3094

3195

3296
class sdist(_sdist):
33-
def run(self):
34-
install_regexes()
35-
_sdist.run(self)
97+
sub_commands = _sdist.sub_commands + [('build_regexes', None)]
98+
99+
100+
cmdclass = {
101+
'develop': develop,
102+
'sdist': sdist,
103+
'build_regexes': build_regexes,
104+
}
36105

37106

38107
setup(
@@ -47,13 +116,9 @@ def run(self):
47116
zip_safe=False,
48117
url='https://github.com/ua-parser/uap-python',
49118
include_package_data=True,
50-
package_data={'ua_parser': ['regexes.json']},
51119
setup_requires=['pyyaml'],
52120
install_requires=[],
53-
cmdclass={
54-
'develop': develop,
55-
'sdist': sdist,
56-
},
121+
cmdclass=cmdclass,
57122
classifiers=[
58123
'Development Status :: 4 - Beta',
59124
'Environment :: Web Environment',

0 commit comments

Comments
 (0)