Split web-monitoring-diff out from -processing

This commit caps off the process of extracting the `web-monitoring-diff` package from the old `web-monitoring-processing` repo! There's a lot happening here:

- The README has been almost entirely rewritten.
- `setup.py` has been updated for this package.
- Versioneer has been updated.
- Requirements have been slimmed down to just what's needed for this codebase.
- Requirements can be installed via `pip install web-monitoring-diff` or `pip install web-monitoring-diff[server,dev]`. `requirements.txt` files are not needed except for Docker and "experimental" diffs.
- The server script has been renamed to `web-monitoring-diff-server` to match the package name. It's long, but now that we are treating this as more abstract, I'm leery of abbreviating as much.
- Modules have been moved around and renamed. `web_monitoring_diff.diff.<whatever>` and `web_monitoring_diff.diff_server` both seemed pretty redundant, so we now have:
    - `web_monitoring_diff.<diff_function>` -- all the diff functions are exposed directly at the top level.
    - The actual submodules containing diff functions are named `<whatever>_diff.py`:
        - `basic_diffs.py` instead of `differs` to be more clear that these are relatively simple functions all thrown together.
        - `html_render_diff.py` instead of `html_diff_render.py` to fit the convention.
        - `html_links_diff.py` instead of `links_diff.py` to clarify that this is for HTML documents.
        - `experimental/<wrapped_package_name>.py` contains the diffs that are no longer actively used and were never especially well supported. They have to be installed by via git instead of PyPI, so setup is also kind of special. Since they need extra special installation support, importing them may fail, and so each one is in a separate module. I've also tried to document the issues and concerns around each of them much more than we had anywhere before.
    - `web_monitoring_diff.exceptions` as a common standard for where our exception types belong.
    - `web_monitoring_diff.server` for the server. It's a subpackage since the content of the server module is so big. I think it's a pretty obvious future plan to refactor that megamodule into smaller files and this lets us do so.
- `deployment.md` has been removed -- it's no longer relevant.
- Switch to flake8 for linting

Layer dependencies better in Dockerfile

Fix CircleCI dependencies

Switch to flake8 for linting (but limit it a lot)

Author: Mr0grog

.circleci/config.yml

@@ -1,37 +1,39 @@
 version: 2
 jobs:
   build:
-    working_directory: ~/web-monitoring-processing
+    working_directory: ~/web-monitoring-diff
     docker:
       - image: circleci/python:3.7
     steps:
       - checkout
       - restore_cache:
           keys:
-            - processing-{{ arch }}-{{ checksum "requirements.txt" }}-{{ checksum "dev-requirements.txt" }}
-            - processing-{{ arch }}-
+            - cache-v1-{{ arch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-experimental.txt" }}
+            - cache-v1-{{ arch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-
+            - cache-v1-{{ arch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-
+            - cache-v1-{{ arch }}-{{ checksum "requirements.txt" }}-
+            - cache-v1-{{ arch }}-
 
       # Bundle install dependencies
       - run:
           name: Install Dependencies
           command: |
             python -m venv venv
             . venv/bin/activate
-            pip install -r requirements.txt
-            pip install -r dev-requirements.txt
-            python setup.py install
+            pip install -e .[server,dev] --no-binary lxml
+            pip install -r requirements-experimental.txt
 
       # Store bundle cache
       - save_cache:
-          key: processing-{{ arch }}-{{ checksum "requirements.txt" }}-{{ checksum "dev-requirements.txt" }}
+          key: cache-v1-{{ arch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-experimental.txt" }}
           paths:
             - venv
 
       - run:
           name: Tests
           command: |
             . venv/bin/activate
-            coverage run run_tests.py -v web_monitoring/tests/
+            coverage run -m pytest -vv
       - run:
           name: Coverage
           command: |
@@ -41,24 +43,24 @@ jobs:
           name: Code linting
           command: |
             . venv/bin/activate
-            pyflakes web_monitoring
-      - run:
-          name: Build docs
-          command: |
-            . venv/bin/activate
-            cd docs && make html
+            flake8 .
+    #   - run:
+    #       name: Build docs
+    #       command: |
+    #         . venv/bin/activate
+    #         cd docs && make html
 
   build_docker:
     machine: true
     steps:
       - checkout
       - run: |
-          docker build -t envirodgi/processing:$CIRCLE_SHA1 .
+          docker build -t envirodgi/web-monitoring-diff:$CIRCLE_SHA1 .
       - run:
           name: Save Image
           command: |
             mkdir /tmp/workspace
-            docker save --output /tmp/workspace/docker-image envirodgi/processing:$CIRCLE_SHA1
+            docker save --output /tmp/workspace/docker-image envirodgi/web-monitoring-diff:$CIRCLE_SHA1
       - persist_to_workspace:
           root: /tmp/workspace
           paths:
@@ -74,25 +76,30 @@ jobs:
           command: docker load --input /tmp/workspace/docker-image
       - run:
           name: Re-tag Image
-          command: docker image tag envirodgi/processing:$CIRCLE_SHA1 envirodgi/processing:latest
+          command: docker image tag envirodgi/web-monitoring-diff:$CIRCLE_SHA1 envirodgi/web-monitoring-diff:latest
       - run:
           name: Publish to Docker Hub
           command: |
             docker login -u $DOCKER_USER -p $DOCKER_PASS
-            docker push envirodgi/processing:$CIRCLE_SHA1
-            docker push envirodgi/processing:latest
+            docker push envirodgi/web-monitoring-diff:$CIRCLE_SHA1
+            docker push envirodgi/web-monitoring-diff:latest
 
 workflows:
   version: 2
   build:
     jobs:
       - build
       - build_docker
-      - publish_docker:
-          requires:
-            - build
-            - build_docker
-          filters:
-            branches:
-              only:
-                - release
+      # XXX: Temporarily disable while in the middle of extracting this package
+      # For this repo, we should probably publish based on tags:
+      # https://circleci.com/docs/2.0/workflows/#executing-workflows-for-a-git-tag
+      # Use $CIRCLE_TAG to identify the tag being used:
+      # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
+      # - publish_docker:
+      #     requires:
+      #       - build
+      #       - build_docker
+      #     filters:
+      #       branches:
+      #         only:
+      #           - release

.coveragerc

@@ -1,6 +1,6 @@
 [run]
 source =
-    web_monitoring
+    web_monitoring_diff
 [report]
 omit =
     */python?.?/*

.gitattributes

@@ -1 +1 @@
-web_monitoring/_version.py export-subst
+web_monitoring_diff/_version.py export-subst

.gitignore

@@ -68,8 +68,8 @@ target/
 # pytest
 .pytest_cache
 
-# diff_output
-diff_output_*
+# test_output
+test_output
 
 # environment file with secrets
 .env

Dockerfile

@@ -12,9 +12,11 @@ WORKDIR /app
 # Copy the requirements.txt alone into the container at /app
 # so that they can be cached more aggressively than the rest of the source.
 ADD requirements.txt /app
-
-# Install any needed packages specified in requirements.txt
 RUN pip install --trusted-host pypi.python.org -r requirements.txt
+ADD requirements-server.txt /app
+RUN pip install --trusted-host pypi.python.org -r requirements-server.txt
+ADD requirements-experimental.txt /app
+RUN pip install --trusted-host pypi.python.org -r requirements-experimental.txt
 
 # Copy the rest of the source.
 ADD . /app
@@ -26,4 +28,4 @@ RUN pip install .
 EXPOSE 80
 
 # Run server on port 80 when the container launches.
-CMD ["wm-diffing-server", "80"]
+CMD ["web-monitoring-diff-server", "80"]

MANIFEST.in

@@ -1,4 +1,3 @@
 include versioneer.py
-include web_monitoring/_version.py
-include web_monitoring/tests/cassettes/*
-include web_monitoring/example_data/*
+include web_monitoring_diff/_version.py
+include web_monitoring_diff/example_data/*