Skip to content

Commit 7ff120b

Browse files
committed
Support platforms where multiprocessing does not work
Check if multiprocessing is supported and else set multiprocessing=False. When later creating list of tasks to run we then choose to either run pool.apply_async(function), or to just eval(function). Import test for multiprocessing is taken from https://github.com/pipxproject/pipx/blob/4870ba6bd0164448c0f6b34dd663cba33b396287/src/pipx/commands/list_packages.py#L12. The full error message from trying to run pythontex on android looks like: ``` This is PythonTeX 0.17 Traceback (most recent call last): File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/synchronize.py", line 28, in <module> from _multiprocessing import SemLock, sem_unlink ImportError: cannot import name 'SemLock' from '_multiprocessing' (/data/data/com.termux/files/usr/lib/python3.8/lib-dynload/_multiprocessing.cpython-38.so) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/data/data/com.termux/files/usr/bin/texlive/pythontex", line 62, in <module> pythontex.main() File "/data/data/com.termux/files/usr/share/texlive/texmf-dist/scripts/pythontex/pythontex3.py", line 2811, in main do_multiprocessing(data, temp_data, old_data, engine_dict) File "/data/data/com.termux/files/usr/share/texlive/texmf-dist/scripts/pythontex/pythontex3.py", line 1269, in do_multiprocessing pool = multiprocessing.Pool(jobs) File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/context.py", line 119, in Pool return Pool(processes, initializer, initargs, maxtasksperchild, File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/pool.py", line 191, in __init__ self._setup_queues() File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/pool.py", line 343, in _setup_queues self._inqueue = self._ctx.SimpleQueue() File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/context.py", line 113, in SimpleQueue return SimpleQueue(ctx=self.get_context()) File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/queues.py", line 336, in __init__ self._rlock = ctx.Lock() File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/context.py", line 67, in Lock from .synchronize import Lock File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/synchronize.py", line 30, in <module> raise ImportError("This platform lacks a functioning sem_open" + ImportError: This platform lacks a functioning sem_open implementation, therefore, the required synchronization primitives needed will not function, see issue 3770. ```
1 parent f6ca8c6 commit 7ff120b

File tree

1 file changed

+81
-55
lines changed

1 file changed

+81
-55
lines changed

pythontex/pythontex3.py

Lines changed: 81 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -57,13 +57,19 @@
5757
from collections import defaultdict, OrderedDict, namedtuple
5858
from re import match, sub, search
5959
import subprocess
60-
import multiprocessing
6160
from pygments.styles import get_all_styles
6261
from pythontex_engines import *
6362
import textwrap
6463
import platform
6564
import itertools
6665

66+
try:
67+
import multiprocessing.synchronize # noqa: F401
68+
from multiprocessing import Pool
69+
except ImportError:
70+
multiprocessing = False
71+
72+
6773
if sys.version_info[0] == 2:
6874
try:
6975
import cPickle as pickle
@@ -192,8 +198,9 @@ def process_argv(data, temp_data):
192198
temp_data['hashdependencies'] = False
193199
if args.jobs is None:
194200
try:
195-
jobs = multiprocessing.cpu_count()
201+
jobs = os.cpu_count()
196202
except NotImplementedError:
203+
# Is this exception needed with os.cpu_count?
197204
jobs = 1
198205
temp_data['jobs'] = jobs
199206
else:
@@ -1145,6 +1152,13 @@ def negative_one():
11451152

11461153

11471154

1155+
def eval_task(fun_name, arg_list):
1156+
'''
1157+
Some platforms, like android, does not support multiprocessing. For such
1158+
platforms we fall back to this function when running tasks.
1159+
'''
1160+
arg_tuple = tuple(arg_list)
1161+
return eval(fun_name)(*arg_tuple)
11481162

11491163
def do_multiprocessing(data, temp_data, old_data, engine_dict):
11501164
jobname = data['jobname']
@@ -1269,7 +1283,14 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
12691283
# concurrent processes to a user-specified value for jobs. If the user
12701284
# has not specified a value, then it will be None, and
12711285
# multiprocessing.Pool() will use cpu_count().
1272-
pool = multiprocessing.Pool(jobs)
1286+
if multiprocessing:
1287+
pool = Pool(jobs)
1288+
# Add task to list of tasks to run asynchronously, from function
1289+
# name and list of args.
1290+
# globals()[fun] converts string with function name into function handle
1291+
async_or_eval = lambda fun, args: pool.apply_async(globals()[fun], args)
1292+
else:
1293+
async_or_eval = eval_task
12731294
tasks = []
12741295

12751296
# If verbose, print a list of processes
@@ -1284,39 +1305,39 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
12841305
family = key.split('#')[0]
12851306
# Uncomment the following for debugging, and comment out what follows
12861307
'''run_code(encoding, outputdir,
1287-
workingdir,
1288-
cc_dict_begin[family],
1289-
code_dict[key],
1290-
cc_dict_end[family],
1291-
engine_dict[family].language,
1292-
engine_dict[family].commands,
1293-
engine_dict[family].created,
1294-
engine_dict[family].extension,
1295-
makestderr, stderrfilename,
1296-
code_index_dict[key],
1297-
engine_dict[family].errors,
1298-
engine_dict[family].warnings,
1299-
engine_dict[family].linenumbers,
1300-
engine_dict[family].lookbehind,
1301-
keeptemps, hashdependencies,
1302-
pygments_settings)'''
1303-
tasks.append(pool.apply_async(run_code, [encoding, outputdir,
1304-
workingdir,
1305-
cc_dict_begin[family],
1306-
code_dict[key],
1307-
cc_dict_end[family],
1308-
engine_dict[family].language,
1309-
engine_dict[family].commands,
1310-
engine_dict[family].created,
1311-
engine_dict[family].extension,
1312-
makestderr, stderrfilename,
1313-
code_index_dict[key],
1314-
engine_dict[family].errors,
1315-
engine_dict[family].warnings,
1316-
engine_dict[family].linenumbers,
1317-
engine_dict[family].lookbehind,
1318-
keeptemps, hashdependencies,
1319-
pygments_settings]))
1308+
workingdir,
1309+
cc_dict_begin[family],
1310+
code_dict[key],
1311+
cc_dict_end[family],
1312+
engine_dict[family].language,
1313+
engine_dict[family].commands,
1314+
engine_dict[family].created,
1315+
engine_dict[family].extension,
1316+
makestderr, stderrfilename,
1317+
code_index_dict[key],
1318+
engine_dict[family].errors,
1319+
engine_dict[family].warnings,
1320+
engine_dict[family].linenumbers,
1321+
engine_dict[family].lookbehind,
1322+
keeptemps, hashdependencies,
1323+
pygments_settings)'''
1324+
tasks.append(async_or_eval('run_code', [encoding, outputdir,
1325+
workingdir,
1326+
cc_dict_begin[family],
1327+
code_dict[key],
1328+
cc_dict_end[family],
1329+
engine_dict[family].language,
1330+
engine_dict[family].commands,
1331+
engine_dict[family].created,
1332+
engine_dict[family].extension,
1333+
makestderr, stderrfilename,
1334+
code_index_dict[key],
1335+
engine_dict[family].errors,
1336+
engine_dict[family].warnings,
1337+
engine_dict[family].linenumbers,
1338+
engine_dict[family].lookbehind,
1339+
keeptemps, hashdependencies,
1340+
pygments_settings]))
13201341
if verbose:
13211342
print(' - Code process ' + key.replace('#', ':'))
13221343

@@ -1331,16 +1352,16 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
13311352
cc_dict_end[family], engine_dict[family].startup,
13321353
engine_dict[family].banner,
13331354
engine_dict[family].filename)'''
1334-
tasks.append(pool.apply_async(python_console, [jobname, encoding,
1335-
outputdir, workingdir,
1336-
fvextfile,
1337-
pygments_settings[family] if family in pygments_settings else None,
1338-
cc_dict_begin[family],
1339-
cons_dict[key],
1340-
cc_dict_end[family],
1341-
engine_dict[family].startup,
1342-
engine_dict[family].banner,
1343-
engine_dict[family].filename]))
1355+
tasks.append(async_or_eval('python_console', [jobname, encoding,
1356+
outputdir, workingdir,
1357+
fvextfile,
1358+
pygments_settings[family] if family in pygments_settings else None,
1359+
cc_dict_begin[family],
1360+
cons_dict[key],
1361+
cc_dict_end[family],
1362+
engine_dict[family].startup,
1363+
engine_dict[family].banner,
1364+
engine_dict[family].filename]))
13441365
else:
13451366
print('* PythonTeX error')
13461367
print(' Currently, non-Python consoles are not supported')
@@ -1353,18 +1374,19 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
13531374
# Uncomment the following for debugging
13541375
# do_pygments(encoding, outputdir, fvextfile, pygments_list,
13551376
# pygments_settings, typeset_cache, hashdependencies)
1356-
tasks.append(pool.apply_async(do_pygments, [encoding, outputdir,
1357-
fvextfile,
1358-
pygments_list,
1359-
pygments_settings,
1360-
typeset_cache,
1361-
hashdependencies]))
1377+
tasks.append(async_or_eval('do_pygments', [encoding, outputdir,
1378+
fvextfile,
1379+
pygments_list,
1380+
pygments_settings,
1381+
typeset_cache,
1382+
hashdependencies]))
13621383
if verbose:
13631384
print(' - Pygments process')
13641385

13651386
# Execute the processes
1366-
pool.close()
1367-
pool.join()
1387+
if multiprocessing:
1388+
pool.close()
1389+
pool.join()
13681390

13691391
# Get the outputs of processes
13701392
# Get the files and macros created. Get the number of errors and warnings
@@ -1375,7 +1397,11 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict):
13751397
new_files = False
13761398
messages = []
13771399
for task in tasks:
1378-
result = task.get()
1400+
if multiprocessing:
1401+
result = task.get()
1402+
else:
1403+
result = task
1404+
13791405
if result['process'] == 'code':
13801406
key = result['key']
13811407
files[key].extend(result['files'])

0 commit comments

Comments
 (0)