From da4aea93e34587c73cfb7a8efd251b1cbf998948 Mon Sep 17 00:00:00 2001 From: Krish Ravindranath Date: Thu, 28 Jul 2022 18:18:41 +0000 Subject: [PATCH 1/4] WIP generation scripts and example sf2 --- examples/notochord/example.sf2 | Bin 0 -> 3576 bytes examples/notochord/generate.py | 91 ++++++++++++++++++++ examples/notochord/generate_realtime.py | 108 ++++++++++++++++++++++++ 3 files changed, 199 insertions(+) create mode 100644 examples/notochord/example.sf2 create mode 100644 examples/notochord/generate.py create mode 100644 examples/notochord/generate_realtime.py diff --git a/examples/notochord/example.sf2 b/examples/notochord/example.sf2 new file mode 100644 index 0000000000000000000000000000000000000000..7fb39c07e4ae5ab71cb76f1bdf9a212e61e56fa8 GIT binary patch literal 3576 zcmcIncT`hp7Joqmq!5qGFuI^1U_o$nWE^K? z6hRRI6{JZK)QE@zqf$h00Ye~>94P@54erYfXJ?&V|JXmg=lpWp?|t9Rci%hbe*RQ< z_h}6PNQ&UZ`cMOy*#H1gY3}ssi0D0vpsfHvfas*SNEOhg_-?l$5C~x2#aHf!r?B@X z>m>5P?_t1olK3UFs4L4Lqa7hme;eFPi{XU?|$!UN>h zM5%(6ag*6e;68r?Z2*9*oj}AJC0IQq+tb$8ieM0%svCf)io^0ZUozxG}y@Qa!h_E2oZnzh6NteAP8vEW@f;Y9y zvsG|rQ*yny^ydK$JC#`#!52ci!|d4OVV$AN!ADq0frHE*Mxsxj2hugzv6@7-)HC00 zhBoFI24SYrR%kqW2u;M`3>%C(OkD|@#0@qCdmm>z&m(;rzD`mcm?K4{?--$Ye>&F8Hxq`y+tO-N$UWw&Q z^t|vx{<6oaN#z~tJWb5{;El^L8G?=yp&p|oNG^g13)fkv?FL1v-BLm-{6jYR{;)KB zx_Bx`^n)-_@O}(Gh8}x7W+Jc?3MY-GMDL)|ocD&ZAxMVOCskwUxYlN!8=JD=M-fGc zXK+QBqfWZk6Agk&10-vybe1Ac9o;=Z=vI0NZQge~uzaGJTzKN#KhFf7T9XHyq@A44 zoB!UW;K|RqMImLx>NicE?WAWp!#bi0DGVT~^>66aTV-T#9%B>bh;zN{-s1Tkb-+{L ze$CawsfeUyan2+dodtJTcTIUpW+SZ<9~iA4$m`MR2)}ox>1y4b>iahzRq`vYR)km1 z+_0$9sNvLaXj<2bd*J=Z>j|_c{drcO^2@nFyP>|JdqdJ!2L@RK=X+_-j&$vMJkGai zOKEDV!&X19>@IVXGnrzfFz z;pNOQbyWDKZv3XORX8%PB?ug?9d>-F@&es;vz_01ya7{nyNq`^`Ion6m-0J~XJq3u zR~?2P3O%Hhp_;`zM$C6C__1(fXObSvtj@pZ4`{+}o4hZWzggN8an0$S>ZZ<*y zGlfOLa^Pu5vR>nsXV_?yP(rb#mfaEOVG7{gO~*6U1ASS7;I_~)@C$hsRu4Rz8Qgh=!af+BPC@t2hCR6&&2y0jbT&5d`ox`8CjiJQR@9(S+eib$yrW%qK zILLVI!FBYsj56D2C`A*|Buujrm4LJfa9;G#q&50E_^+dX^w{L^5O)_ftD3QRO;Gc^ zlmD?%p)R=2uo?5HxgYvg08DS}#6jG+JKDK><9_44PV9+wU_$~3WJN*;sz|*;#-B+M zts6fo$e4(gM9t?fPbw#ChHV6p64W%(8m6tStAW*xs!Rsg2P27)#Ucf;c}_1+DkpM zdA#Fw_YA7U-Og!~1sm!IeM zeS>Fv)4E>s6Pi%f{4%p*m5X-@Q!kp994UWY6Wi+E8QUuv9(z+E93J03YWot^)zXTr zioE>#Y~%6GnZTjN!`x%$1sbIeEmi$Sb2Mm)@tAAnuFbLE?GHC8n!FI@%R27! z(V>+vhC%60z+z#Ck*!)A$^JcVe zAxb0Dpwu>%))7(}b0Be^XBwX$F}!P?7uyDcwONl=e6a8hUr2E23@bNI)PaSxlK2|t zW9vD62biLs44+Er+c_Pk8}gRHbL5$QLc3s?1h(@Q8r@&jvj-^hZBua& z&+;D?Z#&hUx2MRm{hD+{?;nn*sZ^(E^pvb@;9ygEW#5hEMw=%~1GPiBy}$9*t6{|@ zg_}ycYHT}&!=*yns8*L+Rr}f1nXeAR3veyixf{l9yMEaJIXyL*&06Cq#zNlEAF`lk(uo|=Ud1}A9zn6*h) zBJ{mfu?C8Fr+fqvV&>8vb!Yek6k3O>q$>>+>WyJVlJ^`n@uqtC4h_+aS69w;)*13i z(l52my5bi)8%R>ma95ErU+LTS&5g5=R>PdrSJX?TOoxGeGIPPOo$3BPxXXgs+S~oT zxa_z9b~h?X6e&s3TtuQ2q6Ax}@KAM_gZNQ#cb;YYh~8;xG`eh{ymGm5X&|>(y}G2Z zt426X8&#`Xoe3|%&b96OoX%#6F*(Td&ST&i_7NZkrS=*A)s|Y4)Afj+69)X4D-o*L zyHgS3JL(TmRHZ;6R>V=Ohl^%%u9HvJHowr3dWwuMj3nK>P+!-#e!!t_uyE%spTV9t z{Q8esh56}12IE8eQ+BrTZTMw7Qpm#26GR$XXan(IvTK=lYYggu!so;FVk2?A({Cr5 zR6d{rRPRq-60($kLKu7)sQNAU>20EhwMW0XbW*W8yGG@O&ft;8s$Z=u?E21*bM8Jo zW^;VQ;9B?LG(ltyz5rHZHNzSTI%$JPTUkD1ywH+;KMRTpyB^nOb!&c&@LP6G zK5q)a@iYOPHxr_`JRfi~%AU=xQQ_=__|0@)Bs=aa+f(R_|Cw3!MKpoKj?`R<$VXXq z5Nkms?2YICKH9G2z({0xoIU8R0ZxBL1FuH5fY1k_1Z)L10lA-afT2YVKox)jryy9+ zCIYJfBalx7WS`|)F41vG$)=!oadV?GsXHi}Eq)KL{7~rB|8#%1R|C;tg_~C5^6EB# z&;?-#f&e1=zbcIeD}4ZFX=QgIYf4%U6Mdz=ALUVFDg4 z5!AMT8DIt8Fytq-4TSu5Uy8?$gAC literal 0 HcmV?d00001 diff --git a/examples/notochord/generate.py b/examples/notochord/generate.py new file mode 100644 index 0000000..1e28635 --- /dev/null +++ b/examples/notochord/generate.py @@ -0,0 +1,91 @@ +## WIP!! offline generate midi and audiofile from notochord + +from notochord import Notochord + +import fire +from typing import List +from mido import Message, MidiFile, MidiTrack, second2tick +import time + + +def save_midifile(nc_notes: List, filename: str='generated.midi') -> bool: + """ + :param nc_notes: list of note events returned from querying notochord + """ + mid = MidiFile(type=1) + inst_track = {} + + for r in nc_notes: + print(r) + inst, pitch, nctime, velocity, end, step = r['instrument'], r['pitch'], r['time'], r['velocity'], r['end'], r['step'] + + # process notochord events for MIDI + program = inst - 1 # instruments in nc are shifted by 1 to allow for start token? + velocity = round(velocity) # discretize velocity + # can increase the time resolution by increasing MidiFile.ticks_per_beat- typical range is [96-480] but can go higher + midi_time = round(second2tick(nctime)) # TODO: specify ticks_per_beat, tempo + + # TODO does fluid synth handle multi tracks? + track = inst_track.get(inst) + if track is None: + # add a new track for each new instrument + track = MidiTrack() + inst_track[inst] = track + mid.tracks.append(track) + + # TODO: handle drum tracks - should be mapped to channel 10 in 1-128 or channel 9 in 0-127 + # have to synchronize to start at same time or try async type=2? maybe program change does this for us + + track.append(Message('program_change', program=inst, time=0)) + note_on_off = 'note_on' if velocity else 'note_off' + track.append(Message(note_on_off, note=pitch, velocity=velocity, time=midi_time)) + + mid.save(filename) + return True + + +def main(checkpoint: str = None) -> None: + + if checkpoint is None: + raise ValueError(f'Checkpoint required but is None. Use --checkpoint to provide the path to the .ckpt file.') + + predictor = Notochord.from_checkpoint(checkpoint) + predictor.eval() + + include_instrument = None + inst = 1 # grand piano? + pitch = 60 # C4 + nctime = 0 + velocity = 100 + unique_inst = set() + notes = [] + + start = time.time() + # r = predictor.feed(inst, pitch, nctime, velocity) + while time.time() - start < 10: + + # Only allow sampling up to 16 instruments- take the first 16 unique instruments + if len(unique_inst) >= 16: + include_instrument = list(unique_inst) + else: + unique_inst.add(inst) + + # r = predictor.query() + r = predictor.feed_query(inst, pitch, nctime, velocity, include_instrument=include_instrument) + # print(r) + inst, pitch, nctime, velocity, end, step = r['instrument'], r['pitch'], r['time'], r['velocity'], r['end'], r['step'] + notes.append(r) + + end = time.time() + print(end - start) + print((end - start) / step) + print(unique_inst) + print(len(unique_inst)) + print(include_instrument) + predictor.reset() + + save_midifile(notes) + + +if __name__=='__main__': + fire.Fire(main) diff --git a/examples/notochord/generate_realtime.py b/examples/notochord/generate_realtime.py new file mode 100644 index 0000000..9284ed2 --- /dev/null +++ b/examples/notochord/generate_realtime.py @@ -0,0 +1,108 @@ +## WIP realtime audio generation from notochord checkpoint + +from notochord import Notochord + +import fire +import fluidsynth +from mido import second2tick +from multiprocessing import Process, Queue +import time + + +def sequence_event(event: dict) -> None: + """ + :param nc_notes: list of note events returned from querying notochord + """ + ticks_per_beat = 1000 + tempo = 120 + + seq = fluidsynth.Sequencer(time_scale=ticks_per_beat) + fs = fluidsynth.Synth() + fs.start() + sfid = fs.sfload("example.sf2") + fs.program_select(0, sfid, 0, 0) + synthID = seq.register_fluidsynth(fs) + + try: + while True: + if q.empty(): + time.sleep(0.0005) + continue + + r = q.get() + inst, pitch, nctime, velocity, end, step = r['instrument'], r['pitch'], r['time'], r['velocity'], r['end'], r['step'] + + # process notochord events for MIDI + # program = inst - 1 # instruments in nc are shifted by 1 to allow for start token + # TODO: convert inst to prog/ channel + # TODO: ignore start token 0 + + # measure fluidsynth delay - bonus points for latency to actual sound production + # measure notochord latency + # build in 100ms buffer like tidal? + + # first- dumb version - time.sleep nctime + # second version- compensate for drift + # third version- ? + + note_off = velocity == 0 + velocity = round(velocity) # discretize velocity + # can increase the time resolution by increasing MidiFile.ticks_per_beat- typical range is [96-480] but can go higher + midi_time = round(second2tick(nctime, ticks_per_beat, tempo)) + + # TODO: handle drum tracks - should be mapped to channel 10 in 1-128 or channel 9 in 0-127 + if note_off: + seq.note_off(time=midi_time, absolute=False, channel=0, key=pitch, velocity=velocity, dest=synthID) + else: + seq.note_on(time=midi_time, absolute=False, channel=0, key=pitch, velocity=velocity, dest=synthID) + + + except KeyboardInterrupt: + print('sequence_event done') + # finally: + + +def generate_from_notochord(q, checkpoint: str) -> None: + predictor = Notochord.from_checkpoint(checkpoint) + predictor.reset() # TODO: look at reset logic + predictor.eval() + + unique_inst = set() + notes = [] + + # optional feed initial note + r = predictor.feed(inst=1, pitch=60, nctime=0, velocity=100, include_instrument=None) + + try: + while True: + # Only allow sampling up to 16 instruments- take the first 16 unique instruments + if len(unique_inst) >= 16: + include_instrument = list(unique_inst) + else: + unique_inst.add(inst) + # use query feed - blacklist certain instruments, set temperature - absolute simplest way + start = time.time() + r = predictor.query_feed(include_instrument=include_instrument) + end = time.time() - start + time.sleep(nctime) + q.put(r) # add event to queue to be played + except KeyboardInterrupt: + print('generate_from_notochord done') + finally: + predictor.reset() + + +def main(checkpoint: str = None) -> None: + + if checkpoint is None: + raise ValueError(f'Checkpoint required but is None. Use --checkpoint to provide the path to the .ckpt file.') + + p1 = Process(target=generate_from_notochord, args=(q, checkpoint,)) + p2 = Process(target=sequence_event, args=(q,)) + p1.start() + p2.start() + + +if __name__=='__main__': + q = Queue() + fire.Fire(main) From 908d32d012ffd6efcfef26f01751344e336119f2 Mon Sep 17 00:00:00 2001 From: Krish Ravindranath Date: Thu, 28 Jul 2022 18:22:21 +0000 Subject: [PATCH 2/4] add todo --- examples/notochord/generate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/notochord/generate.py b/examples/notochord/generate.py index 1e28635..d58b130 100644 --- a/examples/notochord/generate.py +++ b/examples/notochord/generate.py @@ -1,5 +1,7 @@ ## WIP!! offline generate midi and audiofile from notochord +# TODO: add pyfluidsynth to requirements + from notochord import Notochord import fire From d40d7f98b132782ab74cdc2e2dda7349f85f3277 Mon Sep 17 00:00:00 2001 From: Krish Ravindranath Date: Thu, 28 Jul 2022 18:25:00 +0000 Subject: [PATCH 3/4] add todo for latency --- examples/notochord/generate_realtime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/notochord/generate_realtime.py b/examples/notochord/generate_realtime.py index 9284ed2..6631721 100644 --- a/examples/notochord/generate_realtime.py +++ b/examples/notochord/generate_realtime.py @@ -83,8 +83,8 @@ def generate_from_notochord(q, checkpoint: str) -> None: # use query feed - blacklist certain instruments, set temperature - absolute simplest way start = time.time() r = predictor.query_feed(include_instrument=include_instrument) - end = time.time() - start - time.sleep(nctime) + notochord_latency = time.time() - start + time.sleep(nctime) # TODO: factor in notochord_latency q.put(r) # add event to queue to be played except KeyboardInterrupt: print('generate_from_notochord done') From db0aa3ea9a1398f51a04f4f26ebf69381de89260 Mon Sep 17 00:00:00 2001 From: Krish Ravindranath Date: Thu, 28 Jul 2022 18:26:03 +0000 Subject: [PATCH 4/4] factor in notochord latency --- examples/notochord/generate_realtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/notochord/generate_realtime.py b/examples/notochord/generate_realtime.py index 6631721..e65eaf4 100644 --- a/examples/notochord/generate_realtime.py +++ b/examples/notochord/generate_realtime.py @@ -84,7 +84,7 @@ def generate_from_notochord(q, checkpoint: str) -> None: start = time.time() r = predictor.query_feed(include_instrument=include_instrument) notochord_latency = time.time() - start - time.sleep(nctime) # TODO: factor in notochord_latency + time.sleep(min(nctime - notochord_latency, 0)) # TODO: test logic for factoring in notochord_latency q.put(r) # add event to queue to be played except KeyboardInterrupt: print('generate_from_notochord done')