diff --git a/.gitignore b/.gitignore index b6e4761..3d30f57 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,12 @@ dmypy.json # Pyre type checker .pyre/ + +# Upstream git repo +upstream/ +libinjection/libinjection.h +libinjection/libinjection.py +libinjection/libinjection_xss.* +libinjection/libinjection_html5.* +libinjection/libinjection_sqli* +libinjection/libinjection_wrap* diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f3aeae4 --- /dev/null +++ b/Makefile @@ -0,0 +1,54 @@ + +all: build +# +# + +build: upstream libinjection/libinjection_wrap.c + rm -f libinjection.py libinjection.pyc + python setup.py --verbose build --force + +install: build + sudo python setup.py --verbose install + +test-unit: build words.py + python setup.py build_ext --inplace + PYTHON_PATH='.' nosetests -v --with-xunit test_driver.py + +.PHONY: test +test: test-unit + +.PHONY: speed +speed: + ./speedtest.py + +upstream: + [ -d $@ ] || git clone --depth=1 https://github.com/libinjection/libinjection.git upstream + +libinjection/libinjection.h libinjection/libinjection_sqli.h: upstream + cp -f upstream/src/libinjection*.h upstream/src/libinjection*.c libinjection/ + +words.py: Makefile json2python.py upstream + ./json2python.py < upstream/src/sqlparse_data.json > words.py + + +libinjection/libinjection_wrap.c: libinjection/libinjection.i libinjection/libinjection.h libinjection/libinjection_sqli.h + swig -version + swig -py3 -python -builtin -Wall -Wextra libinjection/libinjection.i + + +.PHONY: copy + +libinjection.so: libinjection/libinjection_wrap.c + gcc -std=c99 -Wall -Werror -fpic -c libinjection/libinjection_sqli.c + gcc -std=c99 -Wall -Werror -fpic -c libinjection/libinjection_xss.c + gcc -std=c99 -Wall -Werror -fpic -c libinjection/libinjection_html5.c + gcc -dynamiclib -shared -o libinjection.so libinjection_sqli.o libinjection_xss.o libinjection_html5.o + +clean: + @rm -rf build dist upstream + @rm -f *.pyc *~ *.so *.o + @rm -f nosetests.xml + @rm -f words.py + @rm -f libinjection/*~ libinjection/*.pyc + @rm -f libinjection/libinjection.h libinjection/libinjection_sqli.h libinjection/libinjection_sqli.c libinjection/libinjection_sqli_data.h + @rm -f libinjection/libinjection_wrap.c libinjection/libinjection.py diff --git a/README.md b/README.md index 9eaf0be..d2937aa 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ -# libinjection-py -libInjection Python bindings +# python3-libinjection +libInjection Python3 bindings diff --git a/apitest.py b/apitest.py new file mode 100755 index 0000000..f23f956 --- /dev/null +++ b/apitest.py @@ -0,0 +1,58 @@ +#!/usr/bin/python + +""" +Work-in-progress +""" + +from libinjection import * +from words import words + +print(dir(libinjection)) + +def print_token_string(tok): + """ + returns the value of token, handling opening and closing quote characters + """ + out = '' + if tok.str_open != "\0": + out += tok.str_open + out += tok.val + if tok.str_close != "\0": + out += tok.str_close + return out + +def print_token(tok): + """ + prints a token for use in unit testing + """ + out = '' + out += tok.type + out += ' ' + if tok.type == 's': + out += print_token_string(tok) + elif tok.type == 'v': + vc = tok.count; + if vc == 1: + out += '@' + elif vc == 2: + out += '@@' + out += print_token_string(tok) + else: + out += tok.val + return out + +def lookup(state, stype, keyword): + keyword = keyword.upper() + if stype == 'v': + keyword = '0' + keyword + ch = words.get(keyword, '') + return ch + +sqli = '1 union all select 1 --' + +s = sqli_state() +sqli_init(s, sqli, libinjection.FLAG_QUOTE_NONE | libinjection.FLAG_SQL_ANSI) +sqli_callback(s, lookup) + +while sqli_tokenize(s): + print(print_token(s.current)) diff --git a/json2python.py b/json2python.py new file mode 100755 index 0000000..83fedc6 --- /dev/null +++ b/json2python.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# +# Copyright 2012, 2013 Nick Galbreath +# nickg@client9.com +# BSD License -- see COPYING.txt for details +# + +""" +Converts a libinjection JSON data file to python dict +""" + +def toc(obj): + """ main routine """ + + print(""" +import libinjection + +def lookup(state, stype, keyword): + keyword = keyword.upper() + if stype == libinjection.LOOKUP_FINGERPRINT: + if keyword in fingerprints and libinjection.sqli_not_whitelist(state): + return 'F' + else: + return chr(0) + return words.get(keyword, chr(0)) + +""") + + words = {} + keywords = obj['keywords'] + for k,v in keywords.items(): + words[str(k)] = str(v) + + print('words = {') + for k in sorted(words.keys()): + print("'{0}': '{1}',".format(k, words[k])) + print('}\n') + + + keywords = obj['fingerprints'] + print('fingerprints = set([') + for k in sorted(keywords): + print("'{0}',".format(k.upper())) + print('])') + + return 0 + +if __name__ == '__main__': + import sys + import json + sys.exit(toc(json.load(sys.stdin))) + diff --git a/libinjection/__init__.py b/libinjection/__init__.py new file mode 100644 index 0000000..84d587a --- /dev/null +++ b/libinjection/__init__.py @@ -0,0 +1 @@ +from libinjection import * diff --git a/libinjection/libinjection.i b/libinjection/libinjection.i new file mode 100644 index 0000000..3f279da --- /dev/null +++ b/libinjection/libinjection.i @@ -0,0 +1,81 @@ +/* libinjection.i SWIG interface file */ +%module libinjection +%{ +#include "libinjection.h" +#include "libinjection_sqli.h" +#include + +/* This is the callback function that runs a python function + * + */ +static char libinjection_python_check_fingerprint(sfilter* sf, int lookuptype, const char* word, size_t len) +{ + PyObject *fp; + PyObject *arglist; + PyObject *result; + const char* strtype; + char ch; + + // get sfilter->pattern + // convert to python string + fp = SWIG_InternalNewPointerObj((void*)sf, SWIGTYPE_p_libinjection_sqli_state,0); + + arglist = Py_BuildValue("(Nis#)", fp, lookuptype, word, len); + // call pyfunct with string arg + result = PyObject_CallObject((PyObject*) sf->userdata, arglist); + Py_DECREF(arglist); + if (result == NULL) { + printf("GOT NULL\n"); + // python call has an exception + // pass it back + ch = '\0'; + } else { + // convert value of python call to a char + strtype = PyString_AsString(result); + ch = strtype[0]; + Py_DECREF(result); + } + return ch; +} + +%} +%include "typemaps.i" + +// The C functions all start with 'libinjection_' as a namespace +// We don't need this since it's in the libinjection python package +// i.e. libinjection.libinjection_is_sqli --> libinjection.is_sqli + // +%rename("%(strip:[libinjection_])s") ""; + +// SWIG doesn't natively support fixed sized arrays. +// this typemap converts the fixed size array sfilter.tokevec +// into a list of pointers to stoken_t types. In otherword this code makes this example work +// s = sfilter() +// libinjection_is_sqli(s, "a string",...) +// for i in len(s.pat): +// print s.tokevec[i].val +// + +%typemap(out) stoken_t [ANY] { +int i; +$result = PyList_New($1_dim0); +for (i = 0; i < $1_dim0; i++) { + PyObject *o = SWIG_NewPointerObj((void*)(& $1[i]), SWIGTYPE_p_stoken_t,0); + PyList_SetItem($result,i,o); +} +} + +// automatically append string length into arg array +%apply (char *STRING, size_t LENGTH) { (const char *s, size_t slen) }; + +%typemap(in) (ptr_lookup_fn fn, void* userdata) { + if ($input == Py_None) { + $1 = NULL; + $2 = NULL; + } else { + $1 = libinjection_python_check_fingerprint; + $2 = $input; + } +} +%include "libinjection.h" +%include "libinjection_sqli.h" diff --git a/libinjection/sqli_fingerprints.py b/libinjection/sqli_fingerprints.py new file mode 100644 index 0000000..8881857 --- /dev/null +++ b/libinjection/sqli_fingerprints.py @@ -0,0 +1,4 @@ + +sqli_fingerprints = set([ +'1234' +]) diff --git a/pytest.py b/pytest.py new file mode 100755 index 0000000..fe13279 --- /dev/null +++ b/pytest.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +from libinjection import * + +sqli= "1 UNION ALL SELECT * FROM FOO" + +if False: + s = sfilter() + print(sqli_fingerprint(s, sqli, CHAR_NULL, COMMENTS_ANSI)) + print("----") + +if False: + s = sfilter() + current = stoken_t() + sqli_init(s, sqli, CHAR_NULL, COMMENTS_ANSI) + while sqli_tokenize(s, current): + print(current.type, current.val) + print("----") + +def is_pattern(state): + return sqli_blacklist(state) and sqli_not_whitelist(state) + +s = sfilter() + +if is_sqli(s, sqli, None): + print("IS SQLI") + print(len(s.pat)) + print(s.current.val) + print(s.current.type) + vec = s.tokenvec + for i in range(len(s.pat)): + atoken = vec[i] + print(atoken.type, atoken.val) +else: + print("IS NOT SQLI") diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..eb47024 --- /dev/null +++ b/setup.py @@ -0,0 +1,49 @@ +""" +libinjection module for python + + Copyright 2012, 2013, 2014 Nick Galbreath + nickg@client9.com + BSD License -- see COPYING.txt for details +""" +try: + from setuptools import setup, Extension +except ImportError: + from distutils.core import setup, Extension + +MODULE = Extension( + '_libinjection', [ + 'libinjection/libinjection_wrap.c', + 'libinjection/libinjection_sqli.c', + 'libinjection/libinjection_html5.c', + 'libinjection/libinjection_xss.c' + ], + swig_opts=['-Wextra', '-builtin'], + define_macros = [], + include_dirs = [], + libraries = [], + library_dirs = [], + ) + +setup ( + name = 'libinjection', + version = '3.9.1', + description = 'Wrapper around libinjection c-code to detect sqli', + author = 'Nick Galbreath', + author_email = 'nickg@client9.com', + url = 'https://libinjection.client9.com/', + ext_modules = [MODULE], + packages = ['libinjection'], + long_description = ''' +wrapper around libinjection +''', + classifiers = [ + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Topic :: Database', + 'Topic :: Security', + 'Operating System :: OS Independent', + 'Development Status :: 3 - Alpha', + 'Topic :: Internet :: Log Analysis', + 'Topic :: Internet :: WWW/HTTP' + ] + ) diff --git a/speedtest.py b/speedtest.py new file mode 100755 index 0000000..31e96e2 --- /dev/null +++ b/speedtest.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +from libinjection import sqli_state +from words import * +import time +import sys +if sys.version_info > (3, 7): + from time import process_time as clock +else: + from time import clock as clock + +def lookup_null(state, style, keyword): + return '' + +def lookup_c(state, style, keyword): + return '' + #return sqli_lookup_word(state, style, keyword) + +def lookup_upcase(state, stype, keyword): + if stype == libinjection.LOOKUP_FINGERPRINT: + return words.get('0' + keyword.upper(), '') + else: + return words.get(keyword.upper(), '') + +def main(): + + inputs = ( + "123 LIKE -1234.5678E+2;", + "APPLE 19.123 'FOO' \"BAR\"", + "/* BAR */ UNION ALL SELECT (2,3,4)", + "1 || COS(+0X04) --FOOBAR", + "dog apple @cat banana bar", + "dog apple cat \"banana \'bar", + "102 TABLE CLOTH" + ) + imax = 100000 + + t0 = clock() + sfilter = sqli_state() + for i in range(imax): + s = inputs[i % 7] + sqli_init(sfilter, s, 0) + is_sqli(sfilter) + t1 = clock() + total = imax / (t1 - t0) + print(("python->c TPS = {0}".format(total))) + + t0 = clock() + sfilter = sqli_state() + for i in range(imax): + s = inputs[i % 7] + sqli_init(sfilter, s, 0) + sqli_callback(sfilter, lookup_null) + is_sqli(sfilter) + t1 = clock() + total = imax / (t1 - t0) + print(("python lookup_null TPS = {0}".format(total))) + + t0 = clock() + sfilter = sqli_state() + for i in range(imax): + s = inputs[i % 7] + sqli_init(sfilter, s, 0) + sqli_callback(sfilter, lookup_upcase) + is_sqli(sfilter) + t1 = clock() + total = imax / (t1 - t0) + print(("python lookup_upcase TPS = {0}".format(total))) + + t0 = clock() + sfilter = sqli_state() + for i in range(imax): + s = inputs[i % 7] + sqli_init(sfilter, s, 0) + sqli_callback(sfilter, lookup_c) + is_sqli(sfilter) + t1 = clock() + total = imax / (t1 - t0) + print(("python lookup_c TPS = {0}".format(total))) + + + t0 = clock() + sfilter = sqli_state() + for i in range(imax): + s = inputs[i % 7] + sqli_init(sfilter, s, 0) + sqli_callback(sfilter, lookup) + is_sqli(sfilter) + t1 = clock() + total = imax / (t1 - t0) + print(("python lookup TPS = {0}".format(total))) + + +if __name__ == '__main__': + main() diff --git a/test_driver.py b/test_driver.py new file mode 100755 index 0000000..0991c07 --- /dev/null +++ b/test_driver.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +""" +Test driver +Runs off plain text files, similar to how PHP's test harness works +""" +import os +import glob +from libinjection import * +from words import * + +print(version()) + +def print_token_string(tok): + """ + returns the value of token, handling opening and closing quote characters + """ + out = '' + if tok.str_open != "\0": + out += tok.str_open + out += tok.val + if tok.str_close != "\0": + out += tok.str_close + return out + +def print_token(tok): + """ + prints a token for use in unit testing + """ + out = '' + out += tok.type + out += ' ' + if tok.type == 's': + out += print_token_string(tok) + elif tok.type == 'v': + vc = tok.count; + if vc == 1: + out += '@' + elif vc == 2: + out += '@@' + out += print_token_string(tok) + else: + out += tok.val + return out.strip() + +def toascii(data): + """ + Converts a utf-8 string to ascii. needed since nosetests xunit is not UTF-8 safe + https://github.com/nose-devs/nose/issues/649 + https://github.com/nose-devs/nose/issues/692 + """ + return data + udata = data.decode('utf-8') + return udata.encode('ascii', 'xmlcharrefreplace') + +def readtestdata(filename): + """ + Read a test file and split into components + """ + + state = None + info = { + '--TEST--': '', + '--INPUT--': '', + '--EXPECTED--': '' + } + + for line in open(filename, 'r'): + line = line.rstrip() + if line in ('--TEST--', '--INPUT--', '--EXPECTED--'): + state = line + elif state: + info[state] += line + '\n' + + # remove last newline from input + info['--INPUT--'] = info['--INPUT--'][0:-1] + + return (info['--TEST--'], info['--INPUT--'].strip(), info['--EXPECTED--'].strip()) + +def runtest(testname, flag, sqli_flags): + """ + runs a test, optionally with valgrind + """ + data = readtestdata(os.path.join('../tests', testname)) + + sql_state = sqli_state() + sqli_init(sql_state, data[1], sqli_flags) + sqli_callback(sql_state, lookup) + actual = '' + + if flag == 'tokens': + while sqli_tokenize(sql_state): + actual += print_token(sql_state.current) + '\n'; + actual = actual.strip() + elif flag == 'folding': + num_tokens = sqli_fold(sql_state) + for i in range(num_tokens): + actual += print_token(sqli_get_token(sql_state, i)) + '\n'; + elif flag == 'fingerprints': + ok = is_sqli(sql_state) + if ok: + actual = sql_state.fingerprint + else: + raise RuntimeException("unknown flag") + + actual = actual.strip() + + if actual != data[2]: + print("INPUT: \n" + toascii(data[1])) + print() + print("EXPECTED: \n" + toascii(data[2])) + print() + print("GOT: \n" + toascii(actual)) + assert actual == data[2] + +def test_tokens(): + for testname in sorted(glob.glob('../tests/test-tokens-*.txt')): + testname = os.path.basename(testname) + runtest(testname, 'tokens', libinjection.FLAG_QUOTE_NONE | libinjection.FLAG_SQL_ANSI) + +def test_tokens_mysql(): + for testname in sorted(glob.glob('../tests/test-tokens_mysql-*.txt')): + testname = os.path.basename(testname) + runtest(testname, 'tokens', libinjection.FLAG_QUOTE_NONE | libinjection.FLAG_SQL_MYSQL) + +def test_folding(): + for testname in sorted(glob.glob('../tests/test-folding-*.txt')): + testname = os.path.basename(testname) + runtest(testname, 'folding', libinjection.FLAG_QUOTE_NONE | libinjection.FLAG_SQL_ANSI) + +def test_fingerprints(): + for testname in sorted(glob.glob('../tests/test-sqli-*.txt')): + testname = os.path.basename(testname) + runtest(testname, 'fingerprints', 0) + + +if __name__ == '__main__': + import sys + sys.stderr.write("run using nosetests\n") + sys.exit(1)