add manual pages and fix configuration for tox

.gitignore

@@ -1,20 +1,22 @@
 *__pycache__*
 .venv*
-BUILD*
-src/subaligner.egg-info*
+build*
 dist*
 *.pyc
-.idea
 .vscode
 *.DS_Store*
 .eggs
 tests/subaligner/resource/test.mp4.aac
 tests/subaligner/resource/test.mp4.wav
 tests/subaligner/training.log
 tests/subaligner/output.log
-training.log
-output.log
-program.prof
+*training.log*
+*output.log*
+*program.prof*
+site/build
+site/source/subaligner.rst
+site/source/modules.rst
+*subaligner.egg-info*
 
 # Unit test / coverage reports
 htmlcov/

.readthedocs.yml

@@ -0,0 +1,21 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: site/source/conf.py
+
+# Build documentation with MkDocs
+#mkdocs:
+#  configuration: mkdocs.yml
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats: all
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  version: 3.6

LICENCE → LICENSE

@@ -1,4 +1,6 @@
-Copyright (c) 2019 Xi Bai
+Copyright (c) 2019-present Xi Bai
+
+The MIT License
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build-gzip
+.PHONY: install uninstall build-gzip build-rpm test test-all pydoc coverage manual clean clean-gzip clean-doc clean-manual clean-build clean-pyc clean-test clean-rpm
 
 define BROWSER_PYSCRIPT
 import os, webbrowser, sys
@@ -14,38 +14,37 @@ BROWSER := python -c "$$BROWSER_PYSCRIPT"
 ## The versions of pycaption depended by pycaption and aeneas have no overlapping.
 ## That will fail setup.py so pip install on requirements.txt is needed.
 install:
-	if [ ! -e ".venv" ]; then virtualenv -p python3 .venv; fi
+	if [ ! -e ".venv" ]; then pip3 install virtualenv; virtualenv -p python3 .venv; fi
 	.venv/bin/pip install --upgrade pip setuptools wheel; \
 	cat requirements.txt | xargs -L 1 .venv/bin/pip install; \
 	cat requirements-dev.txt | xargs -L 1 .venv/bin/pip install; \
 	.venv/bin/pip install -e . --ignore-installed
-	cp ./bin/subaligner_1pass /usr/local/bin/subaligner_1pass
-	cp ./bin/subaligner_2pass /usr/local/bin/subaligner_2pass
+	cp ./bin/subaligner_1pass .venv/bin/subaligner_1pass
+	cp ./bin/subaligner_2pass .venv/bin/subaligner_2pass
 
 uninstall:
-	rm -rf /usr/local/bin/subaligner_1pass
-	rm -rf /usr/local/bin/subaligner_2pass
+	rm -f .venv/bin/subaligner_1pass
+	rm -f .venv/bin/subaligner_2pass
 
 build-gzip:
 	mkdir -p dist
-	tar -czf dist/subligner-cli.tar.gz main bin requirements.txt setup.py tox.ini README.rst HISTORY.rst 
+	tar -czf dist/subligner.tar.gz subaligner bin requirements.txt setup.py README.md LICENCE
 
 build-rpm:
 	mkdir -p BUILD RPMS SRPMS SOURCES BUILDROOT
-	tar -czf SOURCES/subligner-cli.tar.gz main requirements.txt
+	tar -czf SOURCES/subligner.tar.gz subaligner bin requirements.txt setup.py README.md LICENCE
 
 test: ## run tests quickly with the default Python
-	if [ ! -e ".venv" ]; then virtualenv -p python3 .venv; fi
+	if [ ! -e ".venv" ]; then pip3 install virtualenv; virtualenv -p python3 .venv; fi
 	.venv/bin/pip install --upgrade pip setuptools wheel; \
 	cat requirements.txt | xargs -L 1 .venv/bin/pip install; \
 	cat requirements-dev.txt | xargs -L 1 .venv/bin/pip install
 	PYTHONPATH=. .venv/bin/python -m unittest discover
-	-.venv/bin/pycodestyle subaligner tests --ignore=E501
+	-.venv/bin/pycodestyle subaligner tests --ignore=E203,E501,W503
 
 test-all: ## run tests on every Python version with tox
-	if [ ! -e ".venv" ]; then virtualenv -p python3 .venv; fi
+	if [ ! -e ".venv" ]; then pip3 install virtualenv; virtualenv -p python3 .venv; fi
 	.venv/bin/pip install --upgrade pip setuptools wheel; \
-	cat requirements.txt | xargs -L 1 .venv/bin/pip install; \
 	cat requirements-dev.txt | xargs -L 1 .venv/bin/pip install
 	.venv/bin/tox
 
@@ -66,11 +65,10 @@ pydoc: clean-doc ## generate pydoc HTML documentation based on docstrings
 	python -m pydoc -w subaligner.models.training.model; mv subaligner.models.training.model.html docs
 	python -m pydoc -w subaligner.models.training.weights; mv subaligner.models.training.weights.html docs
 	python -m pydoc -w subaligner._version; mv subaligner._version.html docs
-
 	$(BROWSER) docs/index.html
 
 coverage: ## check code coverage quickly with the default Python
-	if [ ! -e ".venv" ]; then virtualenv -p python3 .venv; fi
+	if [ ! -e ".venv" ]; then pip3 install virtualenv; virtualenv -p python3 .venv; fi
 	.venv/bin/pip install --upgrade pip setuptools wheel; \
 	cat requirements.txt | xargs -L 1 .venv/bin/pip install; \
 	cat requirements-dev.txt | xargs -L 1 .venv/bin/pip install
@@ -79,13 +77,24 @@ coverage: ## check code coverage quickly with the default Python
 	.venv/bin/coverage html
 	$(BROWSER) htmlcov/index.html
 
-clean: clean-build clean-pyc clean-test clean-rpm clean-doc ## remove all build, test, coverage and Python artifacts
+manual: clean-manual ## generate manual pages
+	if [ ! -e ".venv" ]; then pip3 install virtualenv; virtualenv -p python3 .venv; fi
+	.venv/bin/pip install --upgrade pip setuptools wheel; \
+	cat requirements-docs.txt | xargs -L 1 .venv/bin/pip install; \
+	SPHINXAPIDOC=../.venv/bin/sphinx-apidoc SPHINXBUILD=../.venv/bin/sphinx-build make -C ./site html
+	$(BROWSER) ./site/build/html/index.html
+
+clean: clean-build clean-pyc clean-test clean-rpm clean-doc clean-manual ## remove all build, test, coverage and Python artifacts
 
 clean-gzip:
 	rm -rf dist
 
 clean-doc: ## remove documents
-	rm -rf docs/*.html 
+	rm -rf docs/*.html
+
+clean-manual: ## remove manual pages
+	rm -rf site/build
+	rm -f site/source/subaligner.rst site/source/modules.rst
 
 clean-build: clean-rpm ## remove build artifacts
 	find . -name '*.egg-info' -exec rm -fr {} +
@@ -105,4 +114,4 @@ clean-test: ## remove test and coverage artifacts
 	rm -rf .pytest_cache
 
 clean-rpm:
-	rm -rf BUILD RPMS SRPMS SOURCES BUILDROOT
+	rm -rf BUILD RPMS SRPMS SOURCES BUILDROOT

README.md

@@ -1,6 +1,7 @@
 [![Build Status](https://travis-ci.com/baxtree/subaligner.svg?branch=master)](https://travis-ci.com/baxtree/subaligner) ![Codecov](https://img.shields.io/codecov/c/github/baxtree/subaligner)
-[![Python 3.6](https://img.shields.io/badge/python-3.6-blue.svg)](https://www.python.org/downloads/release/python-360/) [![Python 3.5](https://img.shields.io/badge/python-3.5-blue.svg)](https://www.python.org/downloads/release/python-350/) [![Python 3.4](https://img.shields.io/badge/python-3.4-blue.svg)](https://www.python.org/downloads/release/python-340/)
-
+[![Python 3.6](https://img.shields.io/badge/python-3.6-blue.svg)](https://www.python.org/downloads/release/python-360/) [![Python 3.5](https://img.shields.io/badge/python-3.5-blue.svg)](https://www.python.org/downloads/release/python-350/) [![Python 3.4](https://img.shields.io/badge/python-3.4-blue.svg)](https://www.python.org/downloads/release/python-340/) 
+[![Documentation Status](https://readthedocs.org/projects/subaligner/badge/?version=latest)](https://subaligner.readthedocs.io/en/latest/?badge=latest)
+[![GitHub license](https://img.shields.io/github/license/baxtree/subaligner)](https://github.com/baxtree/subaligner/blob/master/LICENSE)
 
 ## Dependencies
 [FFmpeg](https://www.ffmpeg.org/) and [eSpeak](http://espeak.sourceforge.net/index.html)
@@ -17,7 +18,7 @@ make install && source .venv/bin/activate
 ```
 # Single-stage alignment (high-level shift with lower latency)
 
-(.venv) $ subaligner_1pass -v video.mp4 -s subtitle.srt
+    (.venv) $ subaligner_1pass -v video.mp4 -s subtitle.srt
 ```
 
 ```
@@ -34,13 +35,14 @@ Subtitle: SubRip and TTML
 Video: MP4, WebM, Ogg, 3GP, FLV and MOV 
 
 ## Anatomy
-Subtitles can be out of sync with their companion audiovisual media files for a variety of causes including latency introduced by Speech-To-Text (STT) on live streams or calibration and rectification involving human intervention during post-production.
+Subtitles can be out of sync with their companion audiovisual media files for a variety of causes including latency introduced by Speech-To-Text on live streams or calibration and rectification involving human intervention during post-production.
 
 A model has been trained with synchronised video and subtitle pairs and later used for predicating shifting offsets and directions under the guidance of a two-stage aligning approach. 
 
 First Stage (Global Alignment):
-![](1st_stage.png)
+![](figures/1st_stage.png)
 
 Second Stage (Parallelised Individual Alignment):
-![](2nd_stage.png)
+![](figures/2nd_stage.png)
+
 

requirements-docs.txt

@@ -0,0 +1,2 @@
+sphinx==2.3.1
+sphinx-rtd-theme==0.4.3

setup.py

@@ -3,43 +3,48 @@
 
 import os
 
-from setuptools import setup, find_packages
+from setuptools import setup
 
-with open(os.path.join("subaligner", "_version.py")) as f:
+with open(os.path.join(os.getcwd(), "subaligner", "_version.py")) as f:
     exec(f.read())
 
 with open("README.md") as readme_file:
     readme = readme_file.read()
 
-with open('requirements.txt') as requirements_file:
-    requirements = requirements_file.read().splitlines()
+with open("requirements.txt") as requirements_file:
+    requirements = requirements_file.read().splitlines()[::-1]
 
-with open('requirements-dev.txt') as requirements_dev_file:
-    requirements_dev = requirements_dev_file.read().splitlines()
+with open("requirements-dev.txt") as requirements_dev_file:
+    requirements_dev = requirements_dev_file.read().splitlines()[::-1]
 
 setup(name="subaligner",
       version=__version__,
       author="Xi Bai",
       author_email="[email protected]",
       classifiers=[
-        'License :: OSI Approved :: MIT License',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
       ],
       license="MIT",
-      url="git@bitbucket.org:baxtree/subaligner-cli.git",
-      description="Automatically align an out-of-sync subtitle to its video with DNN",
-      long_description=readme + '\n\n',
-      python_requires='>=3.4',
+      url="git@github.com:baxtree/subaligner.git",
+      description="Automatically aligns an out-of-sync subtitle file to its companion video/audio using Deep Neural Network and Forced Alignment.",
+      long_description=readme + "\n\n",
+      python_requires=">=3.4",
       package_dir={"subaligner": "subaligner"},
-      packages=find_packages("subaligner"),
+      packages=[
+          "subaligner",
+          "subaligner.models.training.model",
+          "subaligner.models.training.weights",
+      ],
       package_data={
           "subaligner.models.training.model": ["model.hdf5"],
           "subaligner.models.training.weights": ["weights.hdf5"]
       },
-      setup_requires=["numpy"],
       install_requires=requirements,
       test_suite="tests.subaligner",
-      tests_require=requirements_dev
+      tests_require=requirements_dev,
+      setup_requires=["numpy>=1.14.1,<1.18.0"],
+      scripts=["bin/subaligner_1pass", "bin/subaligner_2pass"],
 )

site/Makefile

@@ -0,0 +1,26 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SPHINXAPIDOC  ?= sphinx-apidoc
+SOURCEDIR     = source
+BUILDDIR      = build
+INCLUDED      = ../subaligner
+EXCLUDED      = ../subaligner/models
+
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	rm -f "$(SOURCEDIR)/subaligner.rst"
+	rm -f "$(SOURCEDIR)/modules.rst"
+	$(SPHINXAPIDOC) -o "$(SOURCEDIR)" "$(INCLUDED)" "$(EXCLUDED)" && $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

site/source/acknowledgement.rst

@@ -0,0 +1,13 @@
+########################
+Acknowledgement
+########################
+
+
+**Dependencies**::
+
+    librosa
+    tensorflow
+    scikit-learn
+    pycaption
+    pysrt
+    aeneas

site/source/anatomy.rst

@@ -0,0 +1,28 @@
+########################
+Anatomy
+########################
+
+Under the hood, a model has been trained with synchronised video and subtitle pairs and later used for predicating
+shifting offsets and directions under the guidance of a two-stage aligning approach.
+
+The following figure depicts the primary workflow of the first-stage subtitle alignment. It also includes upfront
+network training and later-on subtitle shifting. The data set used for training contains pairs of a video clip and
+a subtitle file with decent start and end time codes. Mel-Frequency Cepstral Coefficients are extracted in parallel
+and fed into a carefully designed DNN incorporating LSTM layers. For subtitle shifting, an out-of-sync subtitle file
+and a target video clip are fed in as input and the output can be either time offsets by which the subtitle should be
+shifted or a ready-to-playback subtitle file with calibrated start and end time. Notably, the shifting illustrated here
+leads to the completion of the first stage alignment (global shifting).
+
+.. image:: ../../figures/1st_stage.png
+   :width: 500
+   :align: center
+   :alt: Illustration on First Stage Alignment
+
+At the second stage, the target video clip and the globally shifted subtitle file will be broken down into evenly
+timed chunks respectively, each pair of which will be fed into the DNN and aligned in parallel (regional shifting) as
+shown in the following figure. And the concatenated subtitle chunks result in the final output.
+
+.. image:: ../../figures/2nd_stage.png
+   :width: 500
+   :align: center
+   :alt: Illustration on Second Stage Alignment