diff --git a/Makefile b/Makefile index d4c1b9b..c44c4c0 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,8 @@ ACTIVATE = source venv/${PYTHON_TOOL}/bin/activate .PHONY: tests venv -TEST_MODULES = ${patsubst tests/%.py,%,$(wildcard tests/test_*.py)} +UNITTEST_MODULES = ${patsubst tests/%.py,%,$(wildcard tests/unittest_*.py)} +INTTEST_MODULES = ${patsubst tests/%.py,%,$(wildcard tests/test_*.py)} ifeq (${NOCOLOUR},) COL_NOTICE = "\\e[35m" @@ -51,14 +52,17 @@ test_level_system: test_level_integration systemtests # Unit tests test individual parse of a small unit. unittests: test_testable ${NOTICE} "Running unit tests" - ${GOOD} "Unit tests passed (we don't have any yet)" + @# Note: We cd into the tests directory, so that we are testing the installed version, not + @# the version in the repository. + ${ACTIVATE} && cd tests && python -munittest -v ${UNITTEST_MODULES} + ${GOOD} "Unit tests passed" # Integration tests check the integration of those units. integrationtests: test_testable ${NOTICE} "Running integration tests" @# Note: We cd into the tests directory, so that we are testing the installed version, not @# the version in the repository. - ${ACTIVATE} && cd tests && python -munittest -v ${TEST_MODULES} + ${ACTIVATE} && cd tests && python -munittest -v ${INTTEST_MODULES} ${GOOD} "Integration tests passed" # System tests check that the way that a user might use it works. diff --git a/README.md b/README.md index 85c88f0..abfab08 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +fuzzit.dev was [acquired](https://about.gitlab.com/press/releases/2020-06-11-gitlab-acquires-peach-tech-and-fuzzit-to-expand-devsecops-offering.html) by GitLab and the new home for this repo is [here](https://gitlab.com/gitlab-org/security-products/analyzers/fuzzers/pythonfuzz) + # pythonfuzz: coverage-guided fuzz testing for python PythonFuzz is coverage-guided [fuzzer](https://developer.mozilla.org/en-US/docs/Glossary/Fuzzing) for testing python packages. @@ -98,8 +100,6 @@ PythonFuzz is a port of [fuzzitdev/jsfuzz](https://github.com/fuzzitdev/jsfuzz) which is in turn heavily based on [go-fuzz](https://github.com/dvyukov/go-fuzz) originally developed by [Dmitry Vyukov's](https://twitter.com/dvyukov). Which is in turn heavily based on [Michal Zalewski](https://twitter.com/lcamtuf) [AFL](http://lcamtuf.coredump.cx/afl/). -For coverage PythonFuzz is using [coverage](https://coverage.readthedocs.io/en/v4.5.x/) instrumentation and coverage library. - ## Contributions Contributions are welcome!:) There are still a lot of things to improve, and tests and features to add. We will slowly post those in the @@ -109,6 +109,8 @@ any unnecessary work is done. ## Trophies -* [python built-in HTMLParser - unhandled exception](https://bugs.python.org/msg355287) +* [python built-in HTMLParser - unhandled exception](https://bugs.python.org/msg355287), [twice](https://bugs.launchpad.net/beautifulsoup/+bug/1883104) +* [CleverCSV - unhandled exceptions](https://github.com/alan-turing-institute/CleverCSV/issues/7) +* [beautifulsoup](https://bugs.launchpad.net/beautifulsoup/+bug/1883264) **Feel free to add bugs that you found with pythonfuzz to this list via pull-request** diff --git a/examples/run_all_examples.py b/examples/run_all_examples.py index 600d6cc..79f28ef 100755 --- a/examples/run_all_examples.py +++ b/examples/run_all_examples.py @@ -1,6 +1,11 @@ #!/usr/bin/env python """ Run all the examples and collect the timings and results. + +SUT: Invocation +Area: Examples run +Class: Functional +Type: System test """ import argparse diff --git a/pythonfuzz/corpus.py b/pythonfuzz/corpus.py index 051adb3..7d95db5 100644 --- a/pythonfuzz/corpus.py +++ b/pythonfuzz/corpus.py @@ -1,6 +1,5 @@ import os import math -import numpy import random import struct import hashlib @@ -9,8 +8,8 @@ INTERESTING8 = [-128, -1, 0, 1, 16, 32, 64, 100, 127] -INTERESTING16 = [-32768, -129, 128, 255, 256, 512, 1000, 1024, 4096, 32767] -INTERESTING32 = [-2147483648, -100663046, -32769, 32768, 65535, 65536, 100663045, 2147483647] +INTERESTING16 = [0, 128, 255, 256, 512, 1000, 1024, 4096, 32767, 65535] +INTERESTING32 = [0, 1, 32768, 65535, 65536, 100663045, 2147483647, 4294967295] # A list of all the mutator clases we have available @@ -49,22 +48,27 @@ def _rand(n): return 0 return random.randint(0, n-1) - @staticmethod - def _choose_len(n): - x = Corpus._rand(100) + @classmethod + def _choose_len(cls, n): + x = cls._rand(100) if x < 90: - return Corpus._rand(min(8, n)) + 1 + return cls._rand(min(8, n)) + 1 elif x < 99: - return Corpus._rand(min(32, n)) + 1 + return cls._rand(min(32, n)) + 1 else: - return Corpus._rand(n) + 1 + return cls._rand(n) + 1 @staticmethod - def copy(src, dst, start_source, start_dst, end_source=None, end_dst=None): - end_source = len(src) if end_source is None else end_source + def copy(dst, src, start_dst, start_src, end_dst=None, end_src=None): + """ + Copy of content from one slice of a source object to a destination object. + + dst and src may be the same object. + """ + end_src = len(src) if end_src is None else end_src end_dst = len(dst) if end_dst is None else end_dst - byte_to_copy = min(end_source-start_source, end_dst-start_dst) - src[start_source:start_source+byte_to_copy] = dst[start_dst:start_dst+byte_to_copy] + byte_to_copy = min(end_src-start_src, end_dst-start_dst) + dst[start_dst:start_dst+byte_to_copy] = src[start_src:start_src+byte_to_copy] def mutate(self, res): """ @@ -87,9 +91,10 @@ def mutate(self, res): return None pos0 = self._rand(len(res)) - pos1 = pos0 + self._choose_len(len(res) - pos0) - self.copy(res, res, pos1, pos0) - return res[:len(res) - (pos1-pos0)] + num_to_remove = self._choose_len(len(res) - pos0) + pos1 = pos0 + num_to_remove + self.copy(res, res, pos0, pos1) + return res[:len(res) - num_to_remove] @register_mutator @@ -102,7 +107,7 @@ def mutate(self, res): n = self._choose_len(10) for k in range(n): res.append(0) - self.copy(res, res, pos, pos+n) + self.copy(res, res, pos+n, pos) for k in range(n): res[pos+k] = self._rand(256) return res @@ -121,11 +126,10 @@ def mutate(self, res): while src == dst: dst = self._rand(len(res)) n = self._choose_len(len(res) - src) - tmp = bytearray(n) - self.copy(res, tmp, src, 0) + tmp = bytearray(res[src:src+n]) for k in range(n): res.append(0) - self.copy(res, res, dst, dst+n) + self.copy(res, res, dst+n, dst) for k in range(n): res[dst+k] = tmp[k] return res @@ -133,6 +137,7 @@ def mutate(self, res): @register_mutator class MutatorCopyBytes(Mutator): + # FIXME: Check how this diffs from DuplicateBytes name = 'Copy a range of bytes' types = set(['byte', 'copy']) @@ -170,6 +175,7 @@ def mutate(self, res): if len(res) == 0: return None pos = self._rand(len(res)) + # We use rand(255) + 1 so that there is no `^ 0` applied to the byte; it always changes. res[pos] ^= self._rand(255) + 1 return res @@ -199,11 +205,8 @@ def mutate(self, res): if len(res) == 0: return None pos = self._rand(len(res)) - v = self._rand(35) + 1 - if bool(random.getrandbits(1)): - res[pos] = numpy.uint8(res[pos]) + numpy.uint8(v) - else: - res[pos] = numpy.uint8(res[pos]) - numpy.uint8(v) + v = self._rand(2**8) + res[pos] = (res[pos] + v) % 256 return res @@ -216,16 +219,14 @@ def mutate(self, res): if len(res) < 2: return None pos = self._rand(len(res) - 1) - v = numpy.uint16(self._rand(35) + 1) - if bool(random.getrandbits(1)): - v = numpy.uint16(0) - v + v = self._rand(2**16) if bool(random.getrandbits(1)): v = struct.pack('>H', v) else: v = struct.pack('I', v) else: v = struct.pack('Q', v) else: v = struct.pack('H', v) else: v = struct.pack('I', v) else: v = struct.pack('= '3' +psutil==5.6.6 functools32==3.2.3.post2; python_version < '3' diff --git a/setup.py b/setup.py index c1a82a2..69bc416 100644 --- a/setup.py +++ b/setup.py @@ -14,8 +14,7 @@ url="https://github.com/fuzzitdev/pythonfuzz", install_requires=[ # WARNING: Keep these values in line with those in requirements.txt - "coverage==4.5.4", - "psutil==5.6.3", + "psutil==5.6.6", "numpy==1.16.6; python_version < '3'", "numpy==1.17.3; python_version >= '3'", "functools32==3.2.3.post2; python_version < '3'", diff --git a/tests/test_crash.py b/tests/test_crash.py index a48e1c1..e19e550 100644 --- a/tests/test_crash.py +++ b/tests/test_crash.py @@ -1,3 +1,12 @@ +""" +Test the fuzzing terminates when a fault is found. + +SUT: Fuzzer +Area: Fault finding +Class: Functional +Type: Integration test +""" + import io import os import unittest diff --git a/tests/test_nocrash.py b/tests/test_nocrash.py index 22fe36a..210c14f 100644 --- a/tests/test_nocrash.py +++ b/tests/test_nocrash.py @@ -1,3 +1,12 @@ +""" +Test the fuzzing terminates when no faults found, at a run limit. + +SUT: Fuzzer +Area: Non-fault operation +Class: Functional +Type: Integration test +""" + import unittest try: diff --git a/tests/unittest_mutators.py b/tests/unittest_mutators.py new file mode 100644 index 0000000..8ed5ae1 --- /dev/null +++ b/tests/unittest_mutators.py @@ -0,0 +1,207 @@ +""" +Test the mutators operate as desired. + +SUT: Corpus +Area: Mutators +Class: Functional +Type: Unit test +""" + +import unittest + +try: + from unittest.mock import patch +except ImportError: + # Python 2 backport of mock + from mock import patch + +import pythonfuzz.corpus as corpus + + +class FakeCorpus(object): + pass + + +class BaseTestMutators(unittest.TestCase): + """ + Test that the mutators objects are doing what we want them to do. + """ + # Subclasses should set this - 'mutator' will be created as part of setup. + mutator_class = None + + def setUp(self): + self.corpus = FakeCorpus() + self.patch_rand = patch('pythonfuzz.corpus.Mutator._rand') + self.mock_rand = self.patch_rand.start() + self.mock_rand.side_effect = [] + # Update the side effects in your subclass + + self.addCleanup(self.patch_rand.stop) + + self.mutator = self.mutator_class(self.corpus) + + +class TestMutatorRemoveRange(BaseTestMutators): + mutator_class = corpus.MutatorRemoveRange + + def test01_empty(self): + # You cannot remove values from an empty input + res = self.mutator.mutate(bytearray(b'')) + self.assertIsNone(res) + + def test02_remove_section(self): + # Check that it removes a sensible range + + # Check that removing at the 2nd position, removing 4 characters leaves the right string. + self.mock_rand.side_effect = [2, 0, 3] + + res = self.mutator.mutate(bytearray(b'1234567890')) + self.assertEqual(res, bytearray(b'127890')) + + +class TestMutatorInsertBytes(BaseTestMutators): + mutator_class = corpus.MutatorInsertBytes + + def test02_insert_bytes(self): + # Check that it inserts sensibly + + # Check that inserting at the 2nd position, adding 4 characters gives us the right string + self.mock_rand.side_effect = [2, 0, 3, 65, 66, 67, 68] + + res = self.mutator.mutate(bytearray(b'123456789')) + self.assertEqual(res, bytearray(b'12ABCD3456789')) + + +class TestMutatorDuplicateBytes(BaseTestMutators): + mutator_class = corpus.MutatorDuplicateBytes + + def test01_empty(self): + # Cannot work with an empty input + res = self.mutator.mutate(bytearray(b'')) + self.assertIsNone(res) + + def test02_duplicate_bytes(self): + # Check that it duplicates + + # Duplicate from offset 2 to offset 5, length 2 + self.mock_rand.side_effect = [2, 5, 0, 1] + + res = self.mutator.mutate(bytearray(b'123456789')) + self.assertEqual(res, bytearray(b'12345346789')) + + +class TestMutatorCopyBytes(BaseTestMutators): + mutator_class = corpus.MutatorDuplicateBytes + + def test01_empty(self): + # Cannot work with an empty input + res = self.mutator.mutate(bytearray(b'')) + self.assertIsNone(res) + + def test02_duplicate_bytes(self): + # Check that it duplicates + + # Duplicate from offset 2 to offset 5, length 2 + self.mock_rand.side_effect = [2, 5, 0, 1] + + res = self.mutator.mutate(bytearray(b'123456789')) + self.assertEqual(res, bytearray(b'12345346789')) + + +class TestMutatorBitFlip(BaseTestMutators): + mutator_class = corpus.MutatorBitFlip + + def test01_empty(self): + # Cannot work with an empty input + res = self.mutator.mutate(bytearray(b'')) + self.assertIsNone(res) + + def test02_flip_bit(self): + # Check that it flips + + # At offset 4, flip bit 3 + self.mock_rand.side_effect = [4, 3] + + res = self.mutator.mutate(bytearray(b'123456789')) + self.assertEqual(res, bytearray(b'1234=6789')) + + +class TestMutatorRandomiseByte(BaseTestMutators): + mutator_class = corpus.MutatorRandomiseByte + + def test01_empty(self): + # Cannot work with an empty input + res = self.mutator.mutate(bytearray(b'')) + self.assertIsNone(res) + + def test02_randomise_byte(self): + # Check that it changes a byte + + # At offset 4, EOR with 65+1 + self.mock_rand.side_effect = [4, 65] + + res = self.mutator.mutate(bytearray(b'123456789')) + self.assertEqual(res, bytearray(b'1234w6789')) + + +class TestMutatorSwapBytes(BaseTestMutators): + mutator_class = corpus.MutatorSwapBytes + + def test01_empty(self): + # Cannot work with an empty input + res = self.mutator.mutate(bytearray(b'')) + self.assertIsNone(res) + + def test02_swap_bytes(self): + # Check that it swaps bytes + + # Swap bytes at 1 and 6 + self.mock_rand.side_effect = [1, 6] + + res = self.mutator.mutate(bytearray(b'123456789')) + self.assertEqual(res, bytearray(b'173456289')) + + +class TestMutatorAddSubByte(BaseTestMutators): + mutator_class = corpus.MutatorAddSubByte + + def test01_empty(self): + # Cannot work with an empty input + res = self.mutator.mutate(bytearray(b'')) + self.assertIsNone(res) + + def test02_add_bytes(self): + # Check that it adds/subs + # FIXME: Not yet implemented - uses a randomised bit for the add/sub + pass + +# FIXME: Also not implemented AddSubShort, AddSubLong, AddSubLongLong +# FIXME: Not yet implemented ReplaceByte, ReplaceShort, ReplaceLong + + +class TestMutatorReplaceDigit(BaseTestMutators): + mutator_class = corpus.MutatorReplaceDigit + + def test01_empty(self): + # Cannot work with an empty input + res = self.mutator.mutate(bytearray(b'')) + self.assertIsNone(res) + + def test02_no_digits(self): + # Cannot work with a string that has no digits + res = self.mutator.mutate(bytearray(b'wibble')) + self.assertIsNone(res) + + def test03_replace_digit(self): + # Check that it replaces a digit + self.mock_rand.side_effect = [0, 5] + + res = self.mutator.mutate(bytearray(b'there are 4 lights')) + self.assertEqual(res, bytearray(b'there are 5 lights')) + + +# FIXME: Not yet implemented: Dictionary insert, Dictionary Append + + +if __name__ == '__main__': + unittest.main()