15
15
from langdetect .lang_detect_exception import LangDetectException
16
16
from toolz import update_in , assoc
17
17
18
- if sys .version >= ' 3.0' :
18
+ if sys .version >= " 3.0" :
19
19
from urllib .parse import urlparse
20
20
else :
21
21
from urlparse import urlparse
24
24
25
25
# https://github.com/matiasb/python-unidiff/blob/master/unidiff/constants.py#L37
26
26
# @@ (source offset, length) (target offset, length) @@ (section header)
27
- RE_HUNK_HEADER = re .compile (
28
- r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?\ @@[ ]?(.*)$" ,
29
- flags = re .MULTILINE )
27
+ RE_HUNK_HEADER = re .compile (r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?\ @@[ ]?(.*)$" , flags = re .MULTILINE )
30
28
31
29
# ensure deterministec language detection
32
30
DetectorFactory .seed = 0
@@ -82,34 +80,29 @@ def chunkify(iterable, chunksize=10000):
82
80
83
81
def ensure_decoded (thing ):
84
82
if not thing :
85
- logger .debug (' ensure_decoded thing is logically False' )
83
+ logger .debug (" ensure_decoded thing is logically False" )
86
84
return None
87
85
if isinstance (thing , (list , dict )):
88
- logger .debug (' ensure_decoded thing is already decoded' )
86
+ logger .debug (" ensure_decoded thing is already decoded" )
89
87
return thing
90
88
single_encoded_dict = double_encoded_dict = None
91
89
try :
92
90
single_encoded_dict = json .loads (thing )
93
91
if isinstance (single_encoded_dict , dict ):
94
- logger .debug (' ensure_decoded thing is single encoded dict' )
92
+ logger .debug (" ensure_decoded thing is single encoded dict" )
95
93
return single_encoded_dict
96
94
elif isinstance (single_encoded_dict , str ):
97
- logger .debug (' ensure_decoded thing is single encoded str' )
95
+ logger .debug (" ensure_decoded thing is single encoded str" )
98
96
if single_encoded_dict == "" :
99
- logger .debug (
100
- 'ensure_decoded thing is single encoded str == ""' )
97
+ logger .debug ('ensure_decoded thing is single encoded str == ""' )
101
98
return None
102
99
else :
103
100
double_encoded_dict = json .loads (single_encoded_dict )
104
- logger .debug (' ensure_decoded thing is double encoded' )
101
+ logger .debug (" ensure_decoded thing is double encoded" )
105
102
return double_encoded_dict
106
103
except Exception as e :
107
- extra = dict (
108
- thing = thing ,
109
- single_encoded_dict = single_encoded_dict ,
110
- double_encoded_dict = double_encoded_dict ,
111
- error = e )
112
- logger .error ('ensure_decoded error' , extra = extra )
104
+ extra = dict (thing = thing , single_encoded_dict = single_encoded_dict , double_encoded_dict = double_encoded_dict , error = e )
105
+ logger .error ("ensure_decoded error" , extra = extra )
113
106
return None
114
107
115
108
@@ -137,31 +130,30 @@ def extract_keys_from_meta(meta, keys):
137
130
elif isinstance (item , (list , tuple )):
138
131
extracted .extend (item )
139
132
else :
140
- logger .warning (' unusual item in meta: %s' , item )
133
+ logger .warning (" unusual item in meta: %s" , item )
141
134
return extracted
142
135
143
136
144
137
def build_comment_url (parent_permlink = None , author = None , permlink = None ):
145
- return '/' .join ([parent_permlink , author , permlink ])
138
+ return "/" .join ([parent_permlink , author , permlink ])
146
139
147
140
148
141
def canonicalize_url (url , ** kwargs ):
149
142
try :
150
143
canonical_url = w3lib .url .canonicalize_url (url , ** kwargs )
151
144
except Exception as e :
152
- logger .warning (' url preparation error' , extra = dict (url = url , error = e ))
145
+ logger .warning (" url preparation error" , extra = dict (url = url , error = e ))
153
146
return None
154
147
if canonical_url != url :
155
- logger .debug (' canonical_url changed %s to %s' , url , canonical_url )
148
+ logger .debug (" canonical_url changed %s to %s" , url , canonical_url )
156
149
try :
157
150
parsed_url = urlparse (canonical_url )
158
151
if not parsed_url .scheme and not parsed_url .netloc :
159
- _log = dict (
160
- url = url , canonical_url = canonical_url , parsed_url = parsed_url )
161
- logger .warning ('bad url encountered' , extra = _log )
152
+ _log = dict (url = url , canonical_url = canonical_url , parsed_url = parsed_url )
153
+ logger .warning ("bad url encountered" , extra = _log )
162
154
return None
163
155
except Exception as e :
164
- logger .warning (' url parse error' , extra = dict (url = url , error = e ))
156
+ logger .warning (" url parse error" , extra = dict (url = url , error = e ))
165
157
return None
166
158
return canonical_url
167
159
@@ -172,7 +164,7 @@ def findall_patch_hunks(body=None):
172
164
173
165
def detect_language (text ):
174
166
if not text or len (text ) < MIN_TEXT_LENGTH_FOR_DETECTION :
175
- logger .debug (' not enough text to perform langdetect' )
167
+ logger .debug (" not enough text to perform langdetect" )
176
168
return None
177
169
try :
178
170
return detect (text )
@@ -186,7 +178,7 @@ def is_comment(item):
186
178
The item can be a Post object or just a raw comment object from the
187
179
blockchain.
188
180
"""
189
- return item [' parent_author' ] != ""
181
+ return item [" parent_author" ] != ""
190
182
191
183
192
184
def time_elapsed (posting_time ):
@@ -202,16 +194,15 @@ def parse_time(block_time):
202
194
"""Take a string representation of time from the blockchain, and parse
203
195
it into datetime object.
204
196
"""
205
- return datetime .strptime (block_time , ' %Y-%m-%dT%H:%M:%S' )
197
+ return datetime .strptime (block_time , " %Y-%m-%dT%H:%M:%S" )
206
198
207
199
208
200
def time_diff (time1 , time2 ):
209
201
return parse_time (time1 ) - parse_time (time2 )
210
202
211
203
212
204
def keep_in_dict (obj , allowed_keys = list ()):
213
- """ Prune a class or dictionary of all but allowed keys.
214
- """
205
+ """Prune a class or dictionary of all but allowed keys."""
215
206
if type (obj ) == dict :
216
207
items = obj .items ()
217
208
else :
@@ -221,8 +212,7 @@ def keep_in_dict(obj, allowed_keys=list()):
221
212
222
213
223
214
def remove_from_dict (obj , remove_keys = list ()):
224
- """ Prune a class or dictionary of specified keys.
225
- """
215
+ """Prune a class or dictionary of specified keys."""
226
216
if type (obj ) == dict :
227
217
items = obj .items ()
228
218
else :
@@ -232,7 +222,7 @@ def remove_from_dict(obj, remove_keys=list()):
232
222
233
223
234
224
def construct_identifier (* args ):
235
- """ Create a post identifier from comment/post object or arguments.
225
+ """Create a post identifier from comment/post object or arguments.
236
226
237
227
Examples:
238
228
@@ -244,21 +234,20 @@ def construct_identifier(*args):
244
234
245
235
if len (args ) == 1 :
246
236
op = args [0 ]
247
- author , permlink = op [' author' ], op [' permlink' ]
237
+ author , permlink = op [" author" ], op [" permlink" ]
248
238
elif len (args ) == 2 :
249
239
author , permlink = args
250
240
else :
251
- raise ValueError (
252
- 'construct_identifier() received unparsable arguments' )
241
+ raise ValueError ("construct_identifier() received unparsable arguments" )
253
242
254
243
# remove the @ sign in case it was passed in by the user.
255
- author = author .replace ('@' , '' )
244
+ author = author .replace ("@" , "" )
256
245
fields = dict (author = author , permlink = permlink )
257
246
return "{author}/{permlink}" .format (** fields )
258
247
259
248
260
- def json_expand (json_op , key_name = ' json' ):
261
- """ Convert a string json object to Python dict in an op. """
249
+ def json_expand (json_op , key_name = " json" ):
250
+ """Convert a string json object to Python dict in an op."""
262
251
if type (json_op ) == dict and key_name in json_op and json_op [key_name ]:
263
252
try :
264
253
return update_in (json_op , [key_name ], json .loads )
@@ -270,9 +259,9 @@ def json_expand(json_op, key_name='json'):
270
259
271
260
def sanitize_permlink (permlink ):
272
261
permlink = permlink .strip ()
273
- permlink = re .sub ("_|\s|\." , "-" , permlink )
274
- permlink = re .sub ("[^\w-]" , "" , permlink )
275
- permlink = re .sub ("[^a-zA-Z0-9-]" , "" , permlink )
262
+ permlink = re .sub (r "_|\s|\." , "-" , permlink )
263
+ permlink = re .sub (r "[^\w-]" , "" , permlink )
264
+ permlink = re .sub (r "[^a-zA-Z0-9-]" , "" , permlink )
276
265
permlink = permlink .lower ()
277
266
return permlink
278
267
@@ -292,53 +281,49 @@ def derive_permlink(title, parent_permlink=None):
292
281
def resolve_identifier (identifier ):
293
282
294
283
# in case the user supplied the @ sign.
295
- identifier = identifier .replace ('@' , '' )
284
+ identifier = identifier .replace ("@" , "" )
296
285
297
- match = re .match ("([\w\-\.]*)/([\w\-]*)" , identifier )
286
+ match = re .match (r "([\w\-\.]*)/([\w\-]*)" , identifier )
298
287
if not hasattr (match , "group" ):
299
288
raise ValueError ("Invalid identifier" )
300
289
return match .group (1 ), match .group (2 )
301
290
302
291
303
292
def fmt_time (t ):
304
- """ Properly Format Time for permlinks
305
- """
293
+ """Properly Format Time for permlinks"""
306
294
return datetime .utcfromtimestamp (t ).strftime ("%Y%m%dt%H%M%S%Z" )
307
295
308
296
309
297
def fmt_time_string (t ):
310
- """ Properly Format Time for permlinks
311
- """
312
- return datetime .strptime (t , '%Y-%m-%dT%H:%M:%S' )
298
+ """Properly Format Time for permlinks"""
299
+ return datetime .strptime (t , "%Y-%m-%dT%H:%M:%S" )
313
300
314
301
315
302
def fmt_time_from_now (secs = 0 ):
316
- """ Properly Format Time that is `x` seconds in the future
303
+ """Properly Format Time that is `x` seconds in the future
317
304
318
- :param int secs: Seconds to go in the future (`x>0`) or the
319
- past (`x<0`)
320
- :return: Properly formated time for Graphene (`%Y-%m-%dT%H:%M:%S`)
321
- :rtype: str
305
+ :param int secs: Seconds to go in the future (`x>0`) or the
306
+ past (`x<0`)
307
+ :return: Properly formated time for Graphene (`%Y-%m-%dT%H:%M:%S`)
308
+ :rtype: str
322
309
323
310
"""
324
- return datetime .utcfromtimestamp (time .time () + int (secs )).strftime (
325
- '%Y-%m-%dT%H:%M:%S' )
311
+ return datetime .utcfromtimestamp (time .time () + int (secs )).strftime ("%Y-%m-%dT%H:%M:%S" )
326
312
327
313
328
314
def env_unlocked ():
329
- """ Check if wallet passphrase is provided as ENV variable. """
330
- return os .getenv (' UNLOCK' , False )
315
+ """Check if wallet passphrase is provided as ENV variable."""
316
+ return os .getenv (" UNLOCK" , False )
331
317
332
318
333
319
# todo remove these
334
320
def strfage (time , fmt = None ):
335
- """ Format time/age
336
- """
321
+ """Format time/age"""
337
322
if not hasattr (time , "days" ): # dirty hack
338
323
now = datetime .utcnow ()
339
324
if isinstance (time , str ):
340
- time = datetime .strptime (time , ' %Y-%m-%dT%H:%M:%S' )
341
- time = ( now - time )
325
+ time = datetime .strptime (time , " %Y-%m-%dT%H:%M:%S" )
326
+ time = now - time
342
327
343
328
d = {"days" : time .days }
344
329
d ["hours" ], rem = divmod (time .seconds , 3600 )
@@ -355,8 +340,7 @@ def strfage(time, fmt=None):
355
340
356
341
357
342
def strfdelta (tdelta , fmt ):
358
- """ Format time/age
359
- """
343
+ """Format time/age"""
360
344
if not tdelta or not hasattr (tdelta , "days" ): # dirty hack
361
345
return None
362
346
@@ -367,7 +351,7 @@ def strfdelta(tdelta, fmt):
367
351
368
352
369
353
def is_valid_account_name (name ):
370
- return re .match (' ^[a-z][a-z0-9\-.]{2,15}$' , name )
354
+ return re .match (r" ^[a-z][a-z0-9\-.]{2,15}$" , name )
371
355
372
356
373
357
def compat_compose_dictionary (dictionary , ** kwargs ):
@@ -394,20 +378,18 @@ def compat_json(data, ignore_dicts=False):
394
378
"""
395
379
# if this is a unicode string, return its string representation
396
380
if isinstance (data , unicode ):
397
- return data .encode (' utf-8' )
381
+ return data .encode (" utf-8" )
398
382
# if this is a list of values, return list of byte-string values
399
383
if isinstance (data , list ):
400
384
return [compat_json (item , ignore_dicts = True ) for item in data ]
401
385
# if this is a dictionary, return dictionary of byte-string keys and values
402
386
# but only if we haven't already byte-string it
403
387
if isinstance (data , dict ) and not ignore_dicts :
404
- return {
405
- compat_json (key , ignore_dicts = True ): compat_json (value , ignore_dicts = True )
406
- for key , value in data .iteritems ()
407
- }
388
+ return {compat_json (key , ignore_dicts = True ): compat_json (value , ignore_dicts = True ) for key , value in data .iteritems ()}
408
389
# if it's anything else, return it in its original form
409
390
return data
410
391
392
+
411
393
def compat_bytes (item , encoding = None ):
412
394
"""
413
395
This method is required because Python 2.7 `bytes` is simply an alias for `str`. Without this method,
@@ -442,7 +424,7 @@ def __bytes__(self):
442
424
:param encoding: optional encoding parameter to handle the Python 3.6 two argument 'bytes' method.
443
425
:return: a bytes object that functions the same across 3.6 and 2.7
444
426
"""
445
- if hasattr (item , ' __bytes__' ):
427
+ if hasattr (item , " __bytes__" ):
446
428
return item .__bytes__ ()
447
429
else :
448
430
if encoding :
@@ -461,7 +443,7 @@ def compat_chr(item):
461
443
:param item: a length 1 string who's `chr` method needs to be invoked
462
444
:return: the unichr code point of the single character string, item
463
445
"""
464
- if sys .version >= ' 3.0' :
446
+ if sys .version >= " 3.0" :
465
447
return chr (item )
466
448
else :
467
449
return unichr (item )
0 commit comments