giga_piano_xl.py

# -*- coding: utf-8 -*-
"""GIGA_Piano_XL.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/github/asigalov61/GIGA-Piano-XL/blob/main/GIGA_Piano_XL.ipynb

# GIGA-Piano XL (ver. 3.0)

***

Powered by tegridy-tools: https://github.com/asigalov61/tegridy-tools

***

Credit for GPT2-RGA code used in this colab goes out @ Sashmark97 https://github.com/Sashmark97/midigen and @ Damon Gwinn https://github.com/gwinndr/MusicTransformer-Pytorch

***

WARNING: This complete implementation is a functioning model of the Artificial Intelligence. Please excercise great humility, care, and respect. https://www.nscai.gov/

***

#### Project Los Angeles

#### Tegridy Code 2022

***

# (NVIDIA GPU CHECK)
"""

#@title nvidia-smi gpu check
!nvidia-smi

"""# (SETUP ENVIRONMENT)"""

#@title Install all dependencies (run only once per session)

!git clone --depth 1 https://github.com/asigalov61/GIGA-Piano-XL

!pip install torch

!pip install tqdm
!pip install matplotlib
!pip install torch-summary

!pip install numpy

!apt install fluidsynth #Pip does not work for some reason. Only apt works
!pip install midi2audio

#@title Import all needed modules

print('Loading needed modules. Please wait...')
import os
import random
import copy
import statistics
from collections import OrderedDict

from tqdm import tqdm

import matplotlib.pyplot as plt
import numpy as np

from torchsummary import summary
from sklearn import metrics
from sklearn.neighbors import NearestNeighbors
from sklearn.metrics import pairwise_distances

print('Loading core modules...')
os.chdir('/content/GIGA-Piano-XL')

import TMIDIX
from GPT2RGAX import *

print('Loading aux modules...')

from midi2audio import FluidSynth
import librosa.display
from IPython.display import Audio, display

from google.colab import files

os.chdir('/content/')

print('Done!')
print('Enjoy!!')

"""# (LOAD MODEL)"""

# Commented out IPython magic to ensure Python compatibility.
#@title Unzip Pre-Trained GIGA-Piano XL Model
# %cd /content/GIGA-Piano-XL/Model

print('=' * 70)
print('Unzipping pre-trained GIGA-Piano XL model...Please wait...')

!cat GIGA-Piano-XL-Trained-Model.zip* > GIGA-Piano-XL-Trained-Model.zip
print('=' * 70)

!unzip -j GIGA-Piano-XL-Trained-Model.zip
print('=' * 70)

print('Done! Enjoy! :)')
print('=' * 70)
# %cd /content/

#@title Load/Reload the model
full_path_to_model_checkpoint = "/content/GIGA-Piano-XL/Model/GIGA_Piano_XL_Trained_Model_138600_steps_1.259_loss.pth" #@param {type:"string"}

#@markdown Model precision option

model_precision = "bfloat16" # @param ["bfloat16", "float16", "float32"]

#@markdown bfloat16 == Third precision/triple speed (if supported, otherwise the model will default to float16)

#@markdown float16 == Half precision/double speed

#@markdown float32 == Full precision/normal speed

plot_tokens_embeddings = True # @param {type:"boolean"}

print('=' * 70)
print('Loading GIGA-Piano XL model...')

torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul
torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn
device_type = 'cuda'

if model_precision == 'bfloat16' and torch.cuda.is_bf16_supported():
  dtype = 'bfloat16'
else:
  dtype = 'float16'

if model_precision == 'float16':
  dtype = 'float16'

if model_precision == 'float32':
  dtype = 'float32'

ptdtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype]
ctx = torch.amp.autocast(device_type=device_type, dtype=ptdtype)

config = GPTConfig(512,
                  2048,
                  dim_feedforward=2048,
                  n_layer=24,
                  n_head=8,
                  n_embd=1024,
                  enable_rpr=True,
                  er_len=2048)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

model = GPT(config)

model = torch.nn.DataParallel(model)

model.load_state_dict(torch.load(full_path_to_model_checkpoint, map_location=device))

model.to(device)

model.eval()

print('Done!')
print('=' * 70)
print('Model will use', dtype, 'precision...')

summary(model)

if plot_tokens_embeddings:
  cos_sim = metrics.pairwise.cosine_similarity(
      model.module.tok_emb.weight.detach().cpu().numpy()
  )
  plt.figure(figsize=(8, 8))
  plt.imshow(cos_sim, cmap="inferno", interpolation="none")
  im_ratio = cos_sim.shape[0] / cos_sim.shape[1]
  plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)
  plt.xlabel("Position")
  plt.ylabel("Position")
  plt.tight_layout()
  plt.plot()
  plt.savefig("/content/GIGA-Piano-XL-Positional-Embeddings-Plot.png", bbox_inches="tight")

"""# (CUSTOM MIDI)"""

#@title Upload your own seed MIDI

#@markdown Press play button to summon MIDI file upload widget below

render_MIDI_to_audio = True # @param {type:"boolean"}

print('=' * 70)

uploaded_MIDI = files.upload()

print('=' * 70)
if list(uploaded_MIDI.keys()):

  score = TMIDIX.midi2ms_score(list(uploaded_MIDI.values())[0])
  f = list(uploaded_MIDI.keys())[0]

  print('Loading custom MIDI file...')

  events_matrix = []

  itrack = 1

  while itrack < len(score):
      for event in score[itrack]:
          if event[0] == 'note' and event[3] != 9:
              events_matrix.append(event)
      itrack += 1

  # Sorting...
  events_matrix.sort(key=lambda x: x[4], reverse=True)
  events_matrix.sort(key=lambda x: x[1])

  # recalculating timings
  for e in events_matrix:
      e[1] = int(e[1] / 10)
      e[2] = int(e[2] / 20)

  # final processing...

  melody = []
  melody_chords = []
  memory_melody_chords = []

  pe = events_matrix[0]
  for e in events_matrix:

      time = max(0, min(126, e[1]-pe[1]))
      dur = max(1, min(126, e[2]))

      ptc = max(1, min(126, e[4]))
      vel = max(1, min(126, e[5]))

      melody_chords.append([time, dur+128, ptc+256, vel+384])

      if time != 0:
        if ptc < 60:
          ptc = (ptc % 12) + 60
        melody.append([time, dur+128, ptc+256, vel+384])

      pe = e

  inputs = []

  for m in melody_chords:
    inputs.extend(m)

  memory_chunks_train = []

  for i in range(0, (len(inputs) // 128)*128, 128): # 32 notes
    memory_chunks_train.append(inputs[i:i+128])

  memory_chunks_test = []

  for i in range(64, (((len(inputs) // 128)-1)*128), 128): # 32 notes
    memory_chunks_test.append(inputs[i:i+128])

  memory_chunks = memory_chunks_train + memory_chunks_test

  print('Done!')
  print('=' * 70)
  print('Composition has', len(melody_chords), 'notes')
  print('Composition has', len(inputs), 'tokens')
  print('Composition has', len(melody), 'melody notes')
  print('Composition has', len(melody)*4, 'melody tokens')
  print('=' * 70)

  out1 = inputs

  if len(out1) != 0:

      song = out1
      song_f = []
      time = 0
      dur = 0
      vel = 0
      pitch = 0
      channel = 0

      son = []

      song1 = []

      for s in song:
        if s > 127:
          son.append(s)

        else:
          if len(son) == 4:
            song1.append(son)
          son = []
          son.append(s)

      for s in song1:

          time += s[0] * 10

          dur = (s[1]-128) * 20

          channel = 0

          pitch = s[2]-256

          vel = s[3]-384

          song_f.append(['note', time, dur, channel, pitch, vel ])

      detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                                output_signature = 'GIGA-Piano XL',
                                                                output_file_name = '/content/GIGA-Piano-XL-Music-Composition',
                                                                track_name='Project Los Angeles',
                                                                list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                                                                )

      print('Done!')

  print('=' * 70)
  print('Displaying resulting composition...')
  print('=' * 70)

  fname = '/content/GIGA-Piano-XL-Music-Composition'

  pr_duration = song_f[-1][1]+song_f[-1][2]

  pianoroll = [[0] * pr_duration for i in range(128)]

  for s in song_f:
    for i in range(s[2]):
      try:
        pianoroll[s[4]][i+s[1]] = s[4]
      except:
        pass

  piano_roll = np.array(pianoroll)

  plt.figure(figsize=(14, 5))
  librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=16, sr=16000, cmap=plt.cm.hot)
  plt.title(fname)

  if render_MIDI_to_audio:
    FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
    display(Audio(str(fname + '.wav'), rate=16000))

"""# (GENERATE)

# (SINGLE BLOCK CONTINUATION)
"""

#@title Single Continuation Block Generator

#@markdown Play with the settings to get different results

custom_MIDI_or_improvisation = True #@param {type:"boolean"}
number_of_prime_notes = 128 #@param {type:"slider", min:32, max:256, step:8}
number_of_tokens_to_generate = 1024 #@param {type:"slider", min:512, max:1920, step:32}
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
number_of_batches = 2 #@param {type:"slider", min:1, max:8, step:1}
show_stats = True #@param {type:"boolean"}

#===================================================================
print('=' * 70)
print('GIGA-Piano XL Music Model Continuation Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Number of prime notes:', number_of_prime_notes)
print('Model temperature:', temperature)
print('Number of batches:', number_of_batches)
print('=' * 70)

if custom_MIDI_or_improvisation:
  inp = inputs[:number_of_prime_notes*4]
else:
  inp = [126, 126+128, 0+256, 0+384]

with ctx:
  rand_seq = model.module.generate_batches(torch.Tensor(inp),
                                            target_seq_length=number_of_tokens_to_generate,
                                            temperature=temperature,
                                            num_batches=number_of_batches,
                                            verbose=show_stats)

out1 = rand_seq[0].cpu().tolist()

print('Done!')

#@title Explore generated continuations
batch_number = 0 #@param {type:"slider", min:0, max:7, step:1}
render_MIDI_to_audio = True # @param {type:"boolean"}

if batch_number >= number_of_batches:
  bn = 0
else:
  bn = batch_number

print('=' * 70)
print('Displaying batch #:',bn )
print('=' * 70)

out1 = rand_seq[bn].cpu().tolist()

if len(out1) != 0:

    song = out1
    song_f = []
    time = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0

    son = []

    song1 = []

    for s in song:
      if s > 127:
        son.append(s)

      else:
        if len(son) == 4:
          song1.append(son)
        son = []
        son.append(s)

    for s in song1:

        time += s[0] * 10

        dur = (s[1]-128) * 20

        channel = 0

        pitch = s[2]-256

        vel = s[3]-384

        song_f.append(['note', time, dur, channel, pitch, vel ])

    detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                              output_signature = 'GIGA-Piano XL',
                                                              output_file_name = '/content/GIGA-Piano-XL-Music-Composition',
                                                              track_name='Project Los Angeles',
                                                              list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                                                              )

    print('Done!')

print('=' * 70)
print('Displaying resulting composition...')
print('=' * 70)

fname = '/content/GIGA-Piano-XL-Music-Composition'

pr_duration = song_f[-1][1]+song_f[-1][2]

pianoroll = [[0] * pr_duration for i in range(128)]

for s in song_f:
  for i in range(s[2]):
    try:
      pianoroll[s[4]][i+s[1]] = s[4]
    except:
      pass

piano_roll = np.array(pianoroll)

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=16, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

if render_MIDI_to_audio:
  FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
  display(Audio(str(fname + '.wav'), rate=16000))

"""# (MULTIPLE BLOCKS CONTINUATIONS)"""

#@title Simple Multiple Continuation Blocks Generator

#@markdown Play with the settings to get different results

#@markdown You can stop the generation at any time to render partial results

custom_MIDI_or_improvisation = True #@param {type:"boolean"}
number_of_prime_notes = 128 #@param {type:"slider", min:8, max:256, step:8}
size_of_continuation_blocks_in_notes = 16 #@param {type:"slider", min:1, max:128, step:1}
number_of_continuation_blocks = 16 #@param {type:"slider", min:1, max:50, step:1}
number_of_memory_tokens = 1024 #@param {type:"slider", min:128, max:2016, step:32}
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
show_stats = False #@param {type:"boolean"}
render_MIDI_to_audio = True # @param {type:"boolean"}

#===================================================================
print('=' * 70)
print('GIGA-Piano XL Music Model Continuation Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Number of prime notes:', number_of_prime_notes)
print('Size of continuation blocks in notes:', size_of_continuation_blocks_in_notes)
print('Number of continuation blocks:', number_of_continuation_blocks)
print('Number of memory tokens:', number_of_memory_tokens)
print('Model temperature:', temperature)

print('=' * 70)
print('Generating...')
print('=' * 70)

out1 = []

if custom_MIDI_or_improvisation:
  out1 = inputs[:number_of_prime_notes*4]
else:
  out1.extend([126, 126+128, 0+256, 0+384])

for i in tqdm(range(number_of_continuation_blocks)):
  try:

    with ctx:
      rand_seq = model.module.generate(torch.Tensor(out1[-number_of_memory_tokens:]),
                                                target_seq_length=len(out1[-number_of_memory_tokens:])+(size_of_continuation_blocks_in_notes*4),
                                                temperature=temperature,
                                                stop_token=512,
                                                verbose=show_stats)

    out = rand_seq[0].cpu().numpy().tolist()

    out1.extend(out[-(size_of_continuation_blocks_in_notes*4):])

  except KeyboardInterrupt:
    print('Stopping generation...')
    break

  except Exception as e:
    print('Error', e)
    break

print('=' * 70)

if len(out1) != 0:

    song = out1
    song_f = []
    time = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0

    son = []

    song1 = []

    for s in song:
      if s > 127:
        son.append(s)

      else:
        if len(son) == 4:
          song1.append(son)
        son = []
        son.append(s)

    for s in song1:

        time += s[0] * 10

        dur = (s[1]-128) * 20

        channel = 0

        pitch = s[2]-256

        vel = s[3]-384

        song_f.append(['note', time, dur, channel, pitch, vel ])

    detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                              output_signature = 'GIGA-Piano XL',
                                                              output_file_name = '/content/GIGA-Piano-XL-Music-Composition',
                                                              track_name='Project Los Angeles',
                                                              list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                                                              )

    print('Done!')

print('=' * 70)
print('Displaying resulting composition...')
print('=' * 70)

fname = '/content/GIGA-Piano-XL-Music-Composition'

pr_duration = song_f[-1][1]+song_f[-1][2]

pianoroll = [[0] * pr_duration for i in range(128)]

for s in song_f:
  for i in range(s[2]):
    try:
      pianoroll[s[4]][i+s[1]] = s[4]
    except:
      pass

piano_roll = np.array(pianoroll)

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=16, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

if render_MIDI_to_audio:
  FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
  display(Audio(str(fname + '.wav'), rate=16000))

#@title Euclidian Distance Chunks Multiple Continuation Blocks Generator

#@markdown Play with the settings to get different results

#@markdown You can stop the generation at any time to render partial results

custom_MIDI_or_improvisation = True #@param {type:"boolean"}
number_of_prime_notes = 128 #@param {type:"slider", min:32, max:256, step:8}
number_of_continuation_chunks = 10 #@param {type:"slider", min:1, max:100, step:1}
number_of_memory_tokens = 512 #@param {type:"slider", min:16, max:1008, step:16}
number_of_batches = 4 #@param {type:"slider", min:1, max:8, step:1}
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
show_stats = False #@param {type:"boolean"}
render_MIDI_to_audio = True # @param {type:"boolean"}

#===================================================================
print('=' * 70)
print('GIGA-Piano XL Music Model Euclidian Distance Chunks Continuation Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Number of prime notes:', number_of_prime_notes)
print('Number of continuation chunks:', number_of_continuation_chunks)
print('Model temperature:', temperature)

print('=' * 70)
print('Generating...')
print('=' * 70)

out1 = []

if custom_MIDI_or_improvisation:
  out1 = inputs[:number_of_prime_notes*4]
else:
  out1.extend([20, 8+128, 60+256, 90+384])

for i in tqdm(range(number_of_continuation_chunks)):
  try:

    with ctx:
      rand_seq = model.module.generate_batches(torch.Tensor(out1[-number_of_memory_tokens:]),
                                                target_seq_length=len(out1[-number_of_memory_tokens:])+128,
                                                temperature=temperature,
                                                num_batches=number_of_batches,
                                                verbose=show_stats)

    out = rand_seq.cpu().tolist()

    chunks = []

    for i in range(len(out)):
      chunk = out[i][-128:]
      chunks.append(chunk)

    min_euc_dist = []

    for j in range(len(chunks)):
      euc_dist = []
      for i in range(len(memory_chunks)):
          euc_dist.append(pairwise_distances([chunks[j]], [memory_chunks[i]])[0][0])

      min_euc_dist.append(min(euc_dist))

    out = chunks[min_euc_dist.index(min(min_euc_dist))]

    out1.extend(out)

  except KeyboardInterrupt:
    print('Stopping generation...')
    break

  except Exception as e:
    print('Error', e)
    break

print('=' * 70)

if len(out1) != 0:

    song = out1
    song_f = []
    time = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0

    son = []

    song1 = []

    for s in song:
      if s > 127:
        son.append(s)

      else:
        if len(son) == 4:
          song1.append(son)
        son = []
        son.append(s)

    for s in song1:

        time += s[0] * 10

        dur = (s[1]-128) * 20

        channel = 0

        pitch = s[2]-256

        vel = s[3]-384

        song_f.append(['note', time, dur, channel, pitch, vel ])

    detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                              output_signature = 'GIGA-Piano XL',
                                                              output_file_name = '/content/GIGA-Piano-XL-Music-Composition',
                                                              track_name='Project Los Angeles',
                                                              list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                                                              )

    print('Done!')

print('=' * 70)
print('Displaying resulting composition...')
print('=' * 70)

fname = '/content/GIGA-Piano-XL-Music-Composition'

pr_duration = song_f[-1][1]+song_f[-1][2]

pianoroll = [[0] * pr_duration for i in range(128)]

for s in song_f:
  for i in range(s[2]):
    try:
      pianoroll[s[4]][i+s[1]] = s[4]
    except:
      pass

piano_roll = np.array(pianoroll)

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=16, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

if render_MIDI_to_audio:
  FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
  display(Audio(str(fname + '.wav'), rate=16000))

#@title KNN Chunks Multiple Continuation Blocks Generator

#@markdown Play with the settings to get different results

#@markdown You can stop the generation at any time to render partial results

custom_MIDI_or_improvisation = True #@param {type:"boolean"}
number_of_prime_notes = 128 #@param {type:"slider", min:16, max:256, step:8}
number_of_continuation_chunks = 10 #@param {type:"slider", min:1, max:100, step:1}
number_of_memory_tokens = 512 #@param {type:"slider", min:16, max:1008, step:16}
number_of_batches = 4 #@param {type:"slider", min:1, max:8, step:1}
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
show_stats = False #@param {type:"boolean"}
render_MIDI_to_audio = True # @param {type:"boolean"}

#===================================================================
print('=' * 70)
print('GIGA-Piano XL Music Model KNN Chunks Continuation Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Number of prime notes:', number_of_prime_notes)
print('Number of continuation chunks:', number_of_continuation_chunks)
print('Number of memory tokens:', number_of_memory_tokens)
print('Model temperature:', temperature)

print('=' * 70)
print('Generating...')
print('=' * 70)

neigh = NearestNeighbors(n_neighbors=1)
neigh.fit(memory_chunks)

out1 = []

if custom_MIDI_or_improvisation:
  out1 = inputs[:number_of_prime_notes*4]
else:
  out1.extend([126, 126+128, 0+256, 0+384])

for i in tqdm(range(number_of_continuation_chunks)):
  try:
    with ctx:
      rand_seq = model.module.generate_batches(torch.Tensor(out1[-number_of_memory_tokens:]),
                                                target_seq_length=len(out1[-number_of_memory_tokens-1:])+128,
                                                temperature=temperature,
                                                num_batches=number_of_batches,
                                                verbose=show_stats)

    out = rand_seq.cpu().tolist()

    chunks = []
    knn_notes = []

    for i in range(len(out)):
      chunk = out[i][-128:]
      chunks.append(chunk)

    scores = [y[0] for y in neigh.kneighbors(chunks)[0].tolist()]

    out = chunks[scores.index(min(scores))]

    out1.extend(out)

  except KeyboardInterrupt:
    print('Stopping generation...')
    break

  except Exception as e:
    print('Error', e)
    break

print('=' * 70)

if len(out1) != 0:

    song = out1
    song_f = []
    time = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0

    son = []

    song1 = []

    for s in song:
      if s > 127:
        son.append(s)

      else:
        if len(son) == 4:
          song1.append(son)
        son = []
        son.append(s)

    for s in song1:

        time += s[0] * 10

        dur = (s[1]-128) * 20

        channel = 0

        pitch = s[2]-256

        vel = s[3]-384

        song_f.append(['note', time, dur, channel, pitch, vel ])

    detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                              output_signature = 'GIGA-Piano XL',
                                                              output_file_name = '/content/GIGA-Piano-XL-Music-Composition',
                                                              track_name='Project Los Angeles',
                                                              list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                                                              )

    print('Done!')

print('=' * 70)
print('Displaying resulting composition...')
print('=' * 70)

fname = '/content/GIGA-Piano-XL-Music-Composition'

pr_duration = song_f[-1][1]+song_f[-1][2]

pianoroll = [[0] * pr_duration for i in range(128)]

for s in song_f:
  for i in range(s[2]):
    try:
      pianoroll[s[4]][i+s[1]] = s[4]
    except:
      pass

piano_roll = np.array(pianoroll)

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=16, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

if render_MIDI_to_audio:
  FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
  display(Audio(str(fname + '.wav'), rate=16000))

"""# (INPAINTING)"""

#@title Custom MIDI Inpaiting / Controlled Generator

#@markdown Play with the settings to get different results

#@markdown You can stop the inpainting at any time to render partial results

control_type = "Time-Duration" #@param ["Time", "Time-Duration", "Time-Duration-Pitch", "Pitch"]
number_of_prime_notes = 64 #@param {type:"slider", min:1, max:256, step:1}
number_of_memory_tokens = 512 #@param {type:"slider", min:64, max:1920, step:32}
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
show_stats = False #@param {type:"boolean"}
render_MIDI_to_audio = True # @param {type:"boolean"}

#===================================================================
print('=' * 70)
print('GIGA-Piano XL Music Model Inpainting / Controlled Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Control type:', control_type)
print('Number of prime notes:', number_of_prime_notes)
print('Number of memory tokens', number_of_memory_tokens)
print('Model temperature:', temperature)

print('=' * 70)
print('Generating...')
print('=' * 70)

out1 = []

for m in melody_chords[:number_of_prime_notes]:
  out1.extend(m)

if control_type == 'Pitch':
  ctrl = 3
  out1 = out1[:-1]

if control_type == 'Time':
  ctrl = 3
  out1.extend(melody_chords[number_of_prime_notes][:1])

if control_type == 'Time-Duration':
  ctrl = 2
  out1.extend(melody_chords[number_of_prime_notes][:2])

if control_type == 'Time-Duration-Pitch':
  ctrl = 1
  out1.extend(melody_chords[number_of_prime_notes][:3])

for i in tqdm(range(number_of_prime_notes+2, len(melody_chords)-1)):
  try:

    with ctx:
      rand_seq = model.module.generate(torch.Tensor(out1[-(number_of_memory_tokens-ctrl):]),
                                                target_seq_length=len(out1[-(number_of_memory_tokens-ctrl):])+ctrl,
                                                temperature=temperature,
                                                stop_token=512,
                                                verbose=False)

    out = rand_seq[0].cpu().tolist()

    if control_type == 'Pitch':
      out1.extend(out[-ctrl:])
      out1.extend([melody_chords[i+1][2]])

    if control_type == 'Time':
      out1.extend(out[-ctrl:])
      out1.extend(melody_chords[i][:1])

    if control_type == 'Time-Duration':
      out1.extend(out[-ctrl:])
      out1.extend(melody_chords[i][:2])

    if control_type == 'Time-Duration-Pitch':
      out1.extend(out[-ctrl:])
      out1.extend(melody_chords[i][:3])

  except KeyboardInterrupt:
    print('Stopping inpainting...')
    break

  except Exception as e:
    print('Error', e)
    break

print('=' * 70)

if len(out1) != 0:

    song = out1
    song_f = []
    time = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0

    son = []

    song1 = []

    for s in song:
      if s > 127:
        son.append(s)

      else:
        if len(son) == 4:
          song1.append(son)
        son = []
        son.append(s)

    for s in song1:

        time += s[0] * 10

        dur = (s[1]-128) * 20

        channel = 0

        pitch = s[2]-256

        vel = s[3]-384

        song_f.append(['note', time, dur, channel, pitch, vel ])

    detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                        output_signature = 'GIGA-Piano XL',
                                                        output_file_name = '/content/GIGA-Piano-XL-Music-Composition',
                                                        track_name='Project Los Angeles',
                                                        list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                                                        )

    print('Done!')

print('=' * 70)
print('Displaying resulting composition...')
print('=' * 70)

fname = '/content/GIGA-Piano-XL-Music-Composition'

pr_duration = song_f[-1][1]+song_f[-1][2]

pianoroll = [[0] * pr_duration for i in range(128)]

for s in song_f:
  for i in range(s[2]):
    try:
      pianoroll[s[4]][i+s[1]] = s[4]
    except:
      pass

piano_roll = np.array(pianoroll)

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=16, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

if render_MIDI_to_audio:
  FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
  display(Audio(str(fname + '.wav'), rate=16000))

#@title Assisted Custom MIDI Inpaiting / Controlled Generator

#@markdown Play with the settings to get different results

#@markdown You can stop the inpainting at any time to render partial results

control_type = "Time-Duration" #@param ["Time", "Time-Duration", "Time-Duration-Pitch", "Pitch"]
number_of_prime_notes = 64 #@param {type:"slider", min:1, max:256, step:1}
number_of_memory_tokens = 512 #@param {type:"slider", min:64, max:1920, step:32}
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
show_stats = False #@param {type:"boolean"}
render_MIDI_to_audio = True # @param {type:"boolean"}

#===================================================================
print('=' * 70)
print('GIGA-Piano XL Music Model Assisted Inpainting / Controlled Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Control type:', control_type)
print('Number of prime notes:', number_of_prime_notes)
print('Number of memory tokens:', number_of_memory_tokens)
print('Model temperature:', temperature)

print('=' * 70)
print('Generating...')
print('=' * 70)

out1 = []

for m in melody_chords[:number_of_prime_notes]:
  out1.extend(m)

if control_type == 'Pitch':
  ctrl = 3
  out1 = out1[:-1]

if control_type == 'Time':
  ctrl = 3
  out1.extend(melody_chords[number_of_prime_notes][:1])

if control_type == 'Time-Duration':
  ctrl = 2
  out1.extend(melody_chords[number_of_prime_notes][:2])

if control_type == 'Time-Duration-Pitch':
  ctrl = 1
  out1.extend(melody_chords[number_of_prime_notes][:3])

for i in tqdm(range(number_of_prime_notes+2, len(melody_chords)-3, 2)):
  try:

    with ctx:
      rand_seq = model.module.generate(torch.Tensor(out1[-(number_of_memory_tokens-ctrl):]),
                                                target_seq_length=len(out1[-(number_of_memory_tokens-ctrl):])+ctrl,
                                                temperature=temperature,
                                                stop_token=512,
                                                verbose=False)

    out = rand_seq[0].cpu().tolist()

    if control_type == 'Time':
      out1.extend(out[-ctrl:])
      out1.extend(melody_chords[i+2][:1])

    if control_type == 'Time-Duration':
      out1.extend(out[-ctrl:])
      out1.extend(melody_chords[i+2][:2])

    if control_type == 'Time-Duration-Pitch':
      out1.extend(out[-ctrl:])
      out1.extend(melody_chords[i+2][:3])

    if control_type == 'Pitch':
      out1.extend(out[-ctrl:])
      out1.extend([i for s in melody_chords[i+1:i+3] for i in s][2:7])

  except KeyboardInterrupt:
    print('Stopping inpainting...')
    break

  except Exception as e:
    print('Error', e)
    break

print('=' * 70)
if len(out1) != 0:

    song = out1
    song_f = []
    time = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0

    son = []

    song1 = []

    for s in song:
      if s > 127:
        son.append(s)

      else:
        if len(son) == 4:
          song1.append(son)
        son = []
        son.append(s)

    for s in song1:

        time += s[0] * 10

        dur = (s[1]-128) * 20

        channel = 0

        pitch = s[2]-256

        vel = s[3]-384

        song_f.append(['note', time, dur, channel, pitch, vel ])

    detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                        output_signature = 'GIGA-Piano XL',
                                                        output_file_name = '/content/GIGA-Piano-XL-Music-Composition',
                                                        track_name='Project Los Angeles',
                                                        list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                                                        )
    print('Done!')

print('=' * 70)
print('Displaying resulting composition...')
print('=' * 70)

fname = '/content/GIGA-Piano-XL-Music-Composition'

pr_duration = song_f[-1][1]+song_f[-1][2]

pianoroll = [[0] * pr_duration for i in range(128)]

for s in song_f:
  for i in range(s[2]):
    try:
      pianoroll[s[4]][i+s[1]] = s[4]
    except:
      pass

piano_roll = np.array(pianoroll)

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=16, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

if render_MIDI_to_audio:
  FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
  display(Audio(str(fname + '.wav'), rate=16000))

"""# (HARMONIZATION)"""

#@title Melody Harmonization Generator

#@markdown Play with the settings to get different results

#@markdown You can stop the harmonization at any time to render partial results

number_of_prime_notes = 8 #@param {type:"slider", min:1, max:64, step:1}
number_of_notes_per_chord = 3 #@param {type:"slider", min:1, max:5, step:1}
random_number_of_notes_per_chord = False #@param {type:"boolean"}
number_of_memory_tokens = 512 #@param {type:"slider", min:64, max:1920, step:32}
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
show_stats = False #@param {type:"boolean"}
render_MIDI_to_audio = True # @param {type:"boolean"}

#===================================================================
print('=' * 70)
print('GIGA-Piano XL Music Model Melody Harmonization Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Number of prime notes:', number_of_prime_notes)
print('Number of memory tokens:', number_of_memory_tokens)
print('Model temperature:', temperature)
print('=' * 70)
print('Generating...')
print('=' * 70)

inp = []

for i in range(number_of_prime_notes):
  inp.extend(melody[i])

for i in tqdm(range(number_of_prime_notes+1, len(melody))):
  try:

    inp.extend(melody[i])


    if random_number_of_notes_per_chord:
      nnpc = random.randint(1, number_of_notes_per_chord)
    else:
      nnpc = number_of_notes_per_chord

    for j in range(nnpc):
      inp.extend([0])

      with ctx:
        rand_seq = model.module.generate(torch.Tensor(inp[-(number_of_memory_tokens-1):]),
                                                  target_seq_length=len(inp[-(number_of_memory_tokens-1):])+3,
                                                  temperature=temperature,
                                                  stop_token=512,
                                                  verbose=show_stats)

      out = rand_seq[0].cpu().numpy().tolist()

      inp.extend(out[-3:])

  except KeyboardInterrupt:
    print('Stopping harmonization...')
    break

  except Exception as e:
    print('Error', e)
    break

print('=' * 70)

out1 = inp

if len(out1) != 0:

    song = out1
    song_f = []
    time = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0

    son = []

    song1 = []

    for s in song:
      if s > 127:
        son.append(s)

      else:
        if len(son) == 4:
          song1.append(son)
        son = []
        son.append(s)

    for s in song1:

        time += s[0] * 10

        dur = (s[1]-128) * 20

        channel = 0

        pitch = s[2]-256

        vel = s[3]-384

        song_f.append(['note', time, dur, channel, pitch, vel ])

    detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                              output_signature = 'GIGA-Piano XL',
                                                              output_file_name = '/content/GIGA-Piano-XL-Music-Composition',
                                                              track_name='Project Los Angeles',
                                                              list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                                                              )
    print('Done!')

print('=' * 70)
print('Displaying resulting composition...')
print('=' * 70)

fname = '/content/GIGA-Piano-XL-Music-Composition'

pr_duration = song_f[-1][1]+song_f[-1][2]

pianoroll = [[0] * pr_duration for i in range(128)]

for s in song_f:
  for i in range(s[2]):
    try:
      pianoroll[s[4]][i+s[1]] = s[4]
    except:
      pass

piano_roll = np.array(pianoroll)

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=16, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

if render_MIDI_to_audio:
  FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
  display(Audio(str(fname + '.wav'), rate=16000))

"""# (NOTES / CHORDS PROGRESSIONS)"""

#@title Notes Progressions Generator

#@markdown Play with the settings to get different results

#@markdown You can stop the generation at any time to render partial results

control_type = "Time" #@param ["Time", "Time-Duration"]
number_of_notes_to_generate = 100 #@param {type:"slider", min:10, max:5000, step:1}
desired_time = 10 #@param {type:"slider", min:1, max:127, step:1}
desired_duration = 9 #@param {type:"slider", min:1, max:127, step:1}
number_of_memory_tokens = 64 #@param {type:"slider", min:64, max:1920, step:32}
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
show_stats = False #@param {type:"boolean"}
render_MIDI_to_audio = True # @param {type:"boolean"}

#===================================================================
print('=' * 70)
print('GIGA-Piano XL Music Model Notes Progressions Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Control Type:', control_type)
print('Number of memory tokens:', number_of_memory_tokens)
print('Model temperature:', temperature)

print('=' * 70)
print('Generating...')
print('=' * 70)

out1 = []

if control_type == 'Time':
  ctrl = 3
  out1.extend([desired_time])

else:
  ctrl = 2
  out1.extend([desired_time])
  out1.extend([desired_duration+256])

for i in tqdm(range(number_of_notes_to_generate)):

  try:

    with ctx:
      rand_seq = model.module.generate(torch.Tensor(out1[-(number_of_memory_tokens-ctrl):]),
                                                target_seq_length=len(out1[-(number_of_memory_tokens-ctrl):])+ctrl,
                                                temperature=temperature,
                                                stop_token=512,
                                                verbose=show_stats)

    out = rand_seq[0].cpu().tolist()

    if control_type == 'Time':

      out1.extend(out[-ctrl:])
      out1.extend([desired_time])

    else:
      out1.extend(out[-ctrl:])
      out1.extend([desired_time])
      out1.extend([desired_duration+256])

  except KeyboardInterrupt:
    print('Stopping generation...')
    break

  except Exception as e:
    print('Error', e)
    break

print('=' * 70)

if len(out1) != 0:

    song = out1
    song_f = []
    time = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0

    son = []

    song1 = []

    for s in song:
      if s > 127:
        son.append(s)

      else:
        if len(son) == 4:
          song1.append(son)
        son = []
        son.append(s)

    for s in song1:

        time += s[0] * 10

        dur = (s[1]-128) * 20

        channel = 0

        pitch = s[2]-256

        vel = s[3]-384

        song_f.append(['note', time, dur, channel, pitch, vel ])

    detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                              output_signature = 'GIGA-Piano XL',
                                                              output_file_name = '/content/GIGA-Piano-XL-Music-Composition',
                                                              track_name='Project Los Angeles',
                                                              list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                                                              )
    print('Done!')

print('=' * 70)
print('Displaying resulting composition...')
print('=' * 70)

fname = '/content/GIGA-Piano-XL-Music-Composition'

pr_duration = song_f[-1][1]+song_f[-1][2]

pianoroll = [[0] * pr_duration for i in range(128)]

for s in song_f:
  for i in range(s[2]):
    try:
      pianoroll[s[4]][i+s[1]] = s[4]
    except:
      pass

piano_roll = np.array(pianoroll)

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=16, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

if render_MIDI_to_audio:
  FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
  display(Audio(str(fname + '.wav'), rate=16000))

#@title Chords Progressions Generator

#@markdown Play with the settings to get different results

#@markdown You can stop the generation at any time to render partial results

control_type = "Time" #@param ["Time", "Time-Duration"]
number_of_chords_to_generate = 40 #@param {type:"slider", min:10, max:200, step:1}
number_of_notes_per_chord = 2 #@param {type:"slider", min:1, max:5, step:1}
random_number_of_notes_per_chord = False #@param {type:"boolean"}
desired_time = 30 #@param {type:"slider", min:1, max:127, step:1}
desired_duration = 20 #@param {type:"slider", min:1, max:127, step:1}
number_of_memory_tokens = 64 #@param {type:"slider", min:64, max:1920, step:32}
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
show_stats = False #@param {type:"boolean"}
render_MIDI_to_audio = True # @param {type:"boolean"}

#===================================================================
print('=' * 70)
print('GIGA-Piano XL Music Model Chords Progressions Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Control Type:', control_type)
print('Number of memory tokens:', number_of_memory_tokens)
print('Model temperature:', temperature)

print('=' * 70)
print('Generating...')
print('=' * 70)

out1 = []

if control_type == 'Time':
  ctrl = 3
  out1.extend([desired_time])

else:
  ctrl = 2
  out1.extend([desired_time])
  out1.extend([desired_duration+256])

nnpc = number_of_notes_per_chord

for i in tqdm(range(number_of_chords_to_generate)):
  try:

    with ctx:
      rand_seq = model.module.generate(torch.Tensor(out1[-(number_of_memory_tokens-ctrl):]),
                                                target_seq_length=len(out1[-(number_of_memory_tokens-ctrl):])+ctrl,
                                                temperature=temperature,
                                                stop_token=512,
                                                verbose=show_stats)

    out = rand_seq[0].cpu().tolist()

    out1.extend(out[-ctrl:])

    if random_number_of_notes_per_chord:
      nnpc = random.randint(1, number_of_notes_per_chord)

    for j in range(nnpc):
      out1.extend([0])

      if control_type == 'Time-Duration':
        out1.extend([desired_duration+256])

      with ctx:
        rand_seq = model.module.generate(torch.Tensor(out1[-(number_of_memory_tokens-ctrl):]),
                                                target_seq_length=len(out1[-(number_of_memory_tokens-ctrl):])+ctrl,
                                                temperature=temperature,
                                                stop_token=512,
                                                verbose=show_stats)

      out = rand_seq[0].cpu().tolist()

      out1.extend(out[-ctrl:])

    if control_type == 'Time':
      out1.extend([desired_time])

    else:
      out1.extend([desired_time])
      out1.extend([desired_duration+256])

  except KeyboardInterrupt:
    print('Stopping generation...')
    break

  except Exception as e:
    print('Error', e)
    break

print('=' * 70)

if len(out1) != 0:

    song = out1
    song_f = []
    time = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0

    son = []

    song1 = []

    for s in song:
      if s > 127:
        son.append(s)

      else:
        if len(son) == 4:
          song1.append(son)
        son = []
        son.append(s)

    for s in song1:

        time += s[0] * 10

        dur = (s[1]-128) * 20

        channel = 0

        pitch = s[2]-256

        vel = s[3]-384

        song_f.append(['note', time, dur, channel, pitch, vel ])

    detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                              output_signature = 'GIGA-Piano XL',
                                                              output_file_name = '/content/GIGA-Piano-XL-Music-Composition',
                                                              track_name='Project Los Angeles',
                                                              list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                                                              )
    print('Done!')

print('=' * 70)
print('Displaying resulting composition...')
print('=' * 70)

fname = '/content/GIGA-Piano-XL-Music-Composition'

pr_duration = song_f[-1][1]+song_f[-1][2]

pianoroll = [[0] * pr_duration for i in range(128)]

for s in song_f:
  for i in range(s[2]):
    try:
      pianoroll[s[4]][i+s[1]] = s[4]
    except:
      pass

piano_roll = np.array(pianoroll)

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=16, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

if render_MIDI_to_audio:
  FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
  display(Audio(str(fname + '.wav'), rate=16000))

"""# Congrats! You did it! :)"""