From db3daa09346271edfa61068993bc02c3f43704cc Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Thu, 28 Apr 2016 13:53:53 +0200 Subject: [PATCH 01/10] oauth.py: Add HTTP proxy support This patch adds support for a HTTP proxy to the geeknote oauth authorization code. The proxy is read from system settings, usually the environment variable https_proxy. Basic proxy authorization is supported. Note that for geeknote to work through a proxy, proxy support for thrift needs to be enabled, too. See https://issues.apache.org/jira/browse/THRIFT-3798 --- geeknote/oauth.py | 51 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/geeknote/oauth.py b/geeknote/oauth.py index 3c44a01..dd173fb 100644 --- a/geeknote/oauth.py +++ b/geeknote/oauth.py @@ -5,7 +5,8 @@ import Cookie import uuid import re -from urllib import urlencode, unquote +import base64 +from urllib import urlencode, unquote, getproxies, proxy_bypass from urlparse import urlparse import out @@ -58,6 +59,26 @@ class GeekNoteAuth(object): incorrectCode = 0 code = None + def __init__(self): + try: + proxy = getproxies()['https'] + except KeyError: + proxy = None + if proxy is None: + self._proxy = None + else: + # This assumes that the proxy is given in URL form. + # A little simpler as _parse_proxy in urllib2.py + self._proxy = urlparse(proxy) + + if proxy is None or not self._proxy.username: + self._proxy_auth = None + else: + user_pass = "%s:%s" % (urlparse.unquote(self._proxy.username), + urlparse.unquote(self._proxy.password)) + self._proxy_auth = { "Proxy-Authorization": + "Basic " + base64.b64encode(user_pass).strip() } + def getTokenRequestData(self, **kwargs): params = { 'oauth_consumer_key': self.consumerKey, @@ -77,9 +98,11 @@ def loadPage(self, url, uri=None, method="GET", params=""): logging.error("Request URL undefined") tools.exitErr() + if not url.startswith("http"): + url = "https://" + url + urlData = urlparse(url) if not uri: - urlData = urlparse(url) - url = urlData.netloc + url = "%s://%s" (urlData.scheme, urlData.netloc) uri = urlData.path + '?' + urlData.query # prepare params, append to uri @@ -97,11 +120,27 @@ def loadPage(self, url, uri=None, method="GET", params=""): if method == "POST": headers["Content-type"] = "application/x-www-form-urlencoded" - logging.debug("Request URL: %s:/%s > %s # %s", url, + if self._proxy is None or proxy_bypass(urlData.hostname): + host = urlData.hostname + port = urlData.port + real_host = real_port = None + else: + host = self._proxy.hostname + port = self._proxy.port + real_host = urlData.hostname + real_port = urlData.port + + logging.debug("Request URL: %s%s > %s # %s", url, uri, unquote(params), headers["Cookie"]) - conn = httplib.HTTPSConnection(url) - conn.request(method, uri, params, headers) + conn = httplib.HTTPSConnection(host, port) + + if real_host is not None: + conn.set_tunnel(real_host, real_port, headers=self._proxy_auth) + if config.DEBUG: + conn.set_debuglevel(1) + + conn.request(method, url + uri, params, headers) response = conn.getresponse() data = response.read() conn.close() From b9f2a984dcf752dae8918c66ff882f7b049b87fa Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Thu, 28 Apr 2016 16:18:25 +0200 Subject: [PATCH 02/10] proxy_support.md: Documentation for proxy support --- proxy_support.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 proxy_support.md diff --git a/proxy_support.md b/proxy_support.md new file mode 100644 index 0000000..ba292e4 --- /dev/null +++ b/proxy_support.md @@ -0,0 +1,45 @@ +HTTP proxy support for geeknote +=============================== + +I recommend to make this work with virtualenv, to avoid overwriting system files. +The important part is to install in the order **thrift, then evernote, then geeknote**. This will make sure that path search order is correct for thrift. + +``` +# Download thrift and geeknote +git clone https://github.com/apache/thrift.git +git clone https://github.com/mwilck/geeknote.git + +# create and enter a virtual environment +virtualenv /var/tmp/geeknote +. /var/tmp/geeknote/bin/activate + +# Apply proxy-support patches for thrift +cd thrift + +## If the patches don't apply, you may need to check out the state that I wrote the patches for: +## git checkout -b proxy e363a34e63 +curl https://issues.apache.org/jira/secure/attachment/12801233/0001-python-THttpClient-Add-support-for-system-proxy-sett.patch | git am +curl https://issues.apache.org/jira/secure/attachment/12801234/0002-Python-THttpClient-Support-proxy-authorization.patch | git am + +# Install thrift from the patched tree +(cd lib/py; python setup.py install) +cd .. + +# Install evernote +pip install evernote + +# Install geeknote +cd geeknote +python setup.py install +``` + +Now `geeknote login`, `geeknote find`, etc. should work behind a proxy if the `http_proxy` environment variable is correctly set. You can now generate a script to activate the virtual environment: + +``` +cat >~/bin/geeknote <<\EOF +#! /bin/bash +. /var/tmp/geeknote/bin/activate +exec geeknote "$@" +EOF +chmod a+x ~/bin/geeknote +``` \ No newline at end of file From 4f3e5712af5e95fa8b9cc75e251a13fc5a280e27 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 20:26:09 +0200 Subject: [PATCH 03/10] out/printLine: fix several outTest failures The previous code wouldn't see the changed sys.stdout in printLine(). This way it works better. --- geeknote/out.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/geeknote/out.py b/geeknote/out.py index 6c3e27b..062e2fd 100644 --- a/geeknote/out.py +++ b/geeknote/out.py @@ -286,7 +286,9 @@ def printDate(timestamp): return datetime.date.strftime(datetime.date.fromtimestamp(timestamp / 1000), "%d.%m.%Y") -def printLine(line, endLine="\n", out=sys.stdout): +def printLine(line, endLine="\n", out=None): + if out is None: + out = sys.stdout message = line + endLine message = tools.stdoutEncode(message) try: From 9a100d91ed5e8352ef6e9ada3a31430a963b42f8 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 20:49:25 +0200 Subject: [PATCH 04/10] outTest: fix error / failure in test_show_note_success() --- tests/outTest.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/outTest.py b/tests/outTest.py index 778333f..716a7ef 100644 --- a/tests/outTest.py +++ b/tests/outTest.py @@ -22,6 +22,9 @@ class UserStub(object): accounting = AccountingStub() +class NoteAttributesStub(object): + pass + class NoteStub(object): title = 'testnote' created = 10000 @@ -29,7 +32,7 @@ class NoteStub(object): content = '##note content' tagNames = ['tag1', 'tag2', 'tag3'] guid = 12345 - + attributes = NoteAttributesStub() class outTestsWithHackedStdout(unittest.TestCase): @@ -106,10 +109,11 @@ def test_show_note_success(self): note = '''################## TITLE ################## testnote =================== META ================== -Created: 01.01.1970 Updated:01.01.1970 \n'''\ -'''----------------- CONTENT ----------------- +Created: 01.01.1970 +Updated: 01.01.1970 +----------------- CONTENT ----------------- Tags: tag1, tag2, tag3 -##note content\n\n\n''' +##note content\n\n''' showNote(NoteStub()) sys.stdout.seek(0) self.assertEquals(sys.stdout.read(), note) From c94eac3ce107313867351af9e4810f413ec6ee39 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 21:01:15 +0200 Subject: [PATCH 05/10] outTest: fix failure in test_failure_message_success --- tests/outTest.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/outTest.py b/tests/outTest.py index 716a7ef..f2e7c46 100644 --- a/tests/outTest.py +++ b/tests/outTest.py @@ -76,9 +76,13 @@ def test_separator_empty_args_success(self): self.assertEquals(sys.stdout.read(), '\n\n') def test_failure_message_success(self): + sav = sys.stderr + buf = StringIO() + sys.stderr = buf failureMessage('fail') - sys.stdout.seek(0) - self.assertEquals(sys.stdout.read(), 'fail\n') + sys.stderr = sav + buf.seek(0) + self.assertEquals(buf.read(), 'fail\n') def test_success_message_success(self): successMessage('success') From 153de4fa0dc1ed09aebfb9d32bed2a51301de273 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 21:28:41 +0200 Subject: [PATCH 06/10] out: fix wrong date conversion in printDate The conversion has been broken since be00692f0277. That patch divided the timestamp effectively by 1000 twice. --- geeknote/out.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/geeknote/out.py b/geeknote/out.py index 062e2fd..222eb6c 100644 --- a/geeknote/out.py +++ b/geeknote/out.py @@ -275,15 +275,6 @@ def rawInput(message, isPass=False): def printDate(timestamp): - # Author @ash-2000 https://github.com/ash-2000 - # Check for crashing when timestamp is 13 digits on python2.7 - # pull request #260 - - if len(str(timestamp)) == 13: - timestamp = int(str(timestamp)[0:-3]) - - # --- - return datetime.date.strftime(datetime.date.fromtimestamp(timestamp / 1000), "%d.%m.%Y") def printLine(line, endLine="\n", out=None): From e181dc7f54a3f33bacadf7ee2127fadfe47c34ed Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 21:30:35 +0200 Subject: [PATCH 07/10] testOut: fix failure in test_print_date Take account of the fact that evernote measures time in milliseconds since 1.1.1970. Together with the previous commit, this fixes the test failure in test_print_date(). --- tests/outTest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/outTest.py b/tests/outTest.py index f2e7c46..90283c0 100644 --- a/tests/outTest.py +++ b/tests/outTest.py @@ -170,4 +170,4 @@ def test_search_result_success(self): self.assertEquals(sys.stdout.read(), result) def test_print_date(self): - self.assertEquals(printDate(1000000), '12.01.1970') + self.assertEquals(printDate(1000000000L), '12.01.1970') From 25993368bb2b3a550a08462c615dcdf15ac0ecb6 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 21:35:28 +0200 Subject: [PATCH 08/10] outTest: fix failures caused by date field width printList() and searchResult() use 18-character wide date fields. outTest hasn't been adapted to this until now. --- tests/outTest.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/outTest.py b/tests/outTest.py index 90283c0..a2a4ee4 100644 --- a/tests/outTest.py +++ b/tests/outTest.py @@ -124,8 +124,8 @@ def test_show_note_success(self): def test_print_list_without_title_success(self): notes_list = '''Total found: 2 - 1 : 01.01.1970 testnote - 2 : 01.01.1970 testnote\n''' + 1 : 01.01.1970 testnote + 2 : 01.01.1970 testnote\n''' printList([NoteStub() for _ in xrange(2)]) sys.stdout.seek(0) self.assertEquals(sys.stdout.read(), notes_list) @@ -133,8 +133,8 @@ def test_print_list_without_title_success(self): def test_print_list_with_title_success(self): notes_list = '''=================== test ================== Total found: 2 - 1 : 01.01.1970 testnote - 2 : 01.01.1970 testnote\n''' + 1 : 01.01.1970 testnote + 2 : 01.01.1970 testnote\n''' printList([NoteStub() for _ in xrange(2)], title='test') sys.stdout.seek(0) self.assertEquals(sys.stdout.read(), notes_list) @@ -142,8 +142,8 @@ def test_print_list_with_title_success(self): def test_print_list_with_urls_success(self): notes_list = '''=================== test ================== Total found: 2 - 1 : 01.01.1970 testnote >>> https://www.evernote.com/Home.action?#n=12345 - 2 : 01.01.1970 testnote >>> https://www.evernote.com/Home.action?#n=12345 + 1 : 01.01.1970 testnote >>> https://www.evernote.com/Home.action?#n=12345 + 2 : 01.01.1970 testnote >>> https://www.evernote.com/Home.action?#n=12345 ''' printList([NoteStub() for _ in xrange(2)], title='test', showUrl=True) sys.stdout.seek(0) @@ -153,8 +153,8 @@ def test_print_list_with_selector_success(self): out.rawInput = lambda x: 2 notes_list = '''=================== test ================== Total found: 2 - 1 : 01.01.1970 testnote - 2 : 01.01.1970 testnote + 1 : 01.01.1970 testnote + 2 : 01.01.1970 testnote 0 : -Cancel-\n''' out.printList([NoteStub() for _ in xrange(2)], title='test', showSelector=True) sys.stdout.seek(0) @@ -163,8 +163,8 @@ def test_print_list_with_selector_success(self): def test_search_result_success(self): result = '''Search request: test Total found: 2 - 1 : 01.01.1970 testnote - 2 : 01.01.1970 testnote\n''' + 1 : 01.01.1970 testnote + 2 : 01.01.1970 testnote\n''' SearchResult([NoteStub() for _ in xrange(2)], 'test') sys.stdout.seek(0) self.assertEquals(sys.stdout.read(), result) From 4964f31f9e466294da9ae8296597d8e42644da7a Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 21:54:01 +0200 Subject: [PATCH 09/10] geeknoteTest: fix error in testError_createSearchRequest1 This must be an assertRaises test. --- tests/geeknoteTest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/geeknoteTest.py b/tests/geeknoteTest.py index 209c3a2..4b116b8 100644 --- a/tests/geeknoteTest.py +++ b/tests/geeknoteTest.py @@ -88,6 +88,5 @@ def test_createSearchRequest2(self): self.assertEqual(testRequest, response) def testError_createSearchRequest1(self): - testRequest = self.notes._createSearchRequest(search="test text", - date="12.31.1999") - self.assertEqual(testRequest, 'exit') + self.assertRaises(SystemExit, self.notes._createSearchRequest, + search="test text", date="12.31.1999") From 495d161ac8d091efd235790c58d41f586380da3a Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 21:54:52 +0200 Subject: [PATCH 10/10] editorTest: Fix failure in test_ENMLToText The output was wrong by one \n. --- tests/editorTest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/editorTest.py b/tests/editorTest.py index 92e54dc..3e0fa8c 100644 --- a/tests/editorTest.py +++ b/tests/editorTest.py @@ -16,7 +16,6 @@ def setUp(self): _Line 2_ **Line 3** - """ self.HTML_TEXT = "

Header 1

Header 2

Line 1

"\ "Line 2

Line 3

"