Skip to content

Commit

Permalink
initial working plugin with MIDI2 port and parameters (aapbareboneplu…
Browse files Browse the repository at this point in the history 
…ginsample)

context (partly): #80

If there is a MIDI2 port and <parameter>s exist, then PluginPreview now tries
to send parameters as MIDI2 Assignable Controller UMPs.
If there isn't MIDI2 port but <parameter>s exist, then it looks for the
port for each parameter, matching by name.
It is to avoid "fixed port index" (which should NOT be fixed) rather than
"fixed parameter index" (which should be fixed).
If there aren't <parameter>s either, then it works in the traditional way
(ports-based).
  • Loading branch information
@atsushieno
atsushieno committed Aug 23, 2022
1 parent 160a725 commit e6eab86
Show file tree
Hide file tree
Showing 6 changed files with 219 additions and 48 deletions.
1 change: 1 addition & 0 deletions androidaudioplugin-samples-host-engine/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ dependencies {
implementation ("androidx.appcompat:appcompat:1.4.1")
implementation ("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.0")
implementation ("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.6.0")
implementation ("dev.atsushieno:ktmidi:0.3.18")
testImplementation ("junit:junit:4.13.2")
androidTestImplementation ("androidx.test.ext:junit:1.1.3")
androidTestImplementation ("androidx.test.espresso:espresso-core:3.4.0")
Expand Down
10 changes: 10 additions & 0 deletions ...amples-host-engine/src/main/java/org/androidaudioplugin/samples/host/engine/MidiHelper.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,16 @@ class MidiHelper {

private fun Int.fromBCD() = ((this and 0xF0) shr 4) * 10 + (this and 0xF)

// see aap-midi2.h for MidiBufferHeader structure.
internal fun toMidiBufferHeader(timeOption: Int, size: UInt): List<Byte> {
// it's not really a UMP but reuse it for encoding to bytes...
return timeOption.toPlatformNativeBytes() + size.toInt().toPlatformNativeBytes() +
List(24) { 0 }
}

private fun Int.toPlatformNativeBytes() = listOf(this % 0x100, this / 0x100 % 0x100,
this / 0x10000 % 0x100, this / 0x1000000).map { it.toByte() }

internal fun toMidiTimeCode(
frameRate: Int,
framesPerSeconds: Int,
Expand Down
88 changes: 76 additions & 12 deletions ...les-host-engine/src/main/java/org/androidaudioplugin/samples/host/engine/PluginPreview.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ import android.content.Context
import android.media.AudioAttributes
import android.media.AudioFormat
import android.media.AudioTrack
import dev.atsushieno.ktmidi.Ump
import dev.atsushieno.ktmidi.UmpFactory
import dev.atsushieno.ktmidi.toPlatformNativeBytes
import org.androidaudioplugin.*
import org.androidaudioplugin.hosting.AudioPluginClient
import org.androidaudioplugin.hosting.AudioPluginInstance
Expand Down Expand Up @@ -99,6 +102,20 @@ class PluginPreview(context: Context) {
errorCallback(instance.proxyError!!)
}

private fun parameterChangesToUMP(parametersFrom0To1: FloatArray) =
(0 until instance!!.getParameterCount()).map { paraI ->
val para = instance!!.getParameter(paraI)
Ump(
UmpFactory.midi2NRPN(
0,
0,
para.id / 0x100,
para.id % 0x100,
parametersFrom0To1[paraI].toBits().toLong()
)
).toPlatformNativeBytes()
}

private fun processPluginOnce(parametersOnUI: FloatArray?) {
val instance = this.instance!!
val parameters = parametersOnUI ?: (0 until instance.getParameterCount()).map { instance.getParameter(it).defaultValue .toFloat() }.toFloatArray()
Expand All @@ -107,17 +124,14 @@ class PluginPreview(context: Context) {
host.resetOutputBuffers()
host.resetControlBuffers()

val midiSequence = MidiHelper.getMidiSequence()
val midi1Events = MidiHelper.splitMidi1Events(midiSequence.toUByteArray())
val midi1EventsGroups = MidiHelper.groupMidi1EventsByTiming(midi1Events).toList()

// Kotlin version of audio/MIDI processing.

var audioInL = -1
var audioInR = -1
var audioOutL = -1
var audioOutR = -1
var midiIn = -1
var midi2In = -1
(0 until instance.getPortCount()).forEach { i ->
val p = instance.getPort(i)
if (p.content == PortInformation.PORT_CONTENT_TYPE_AUDIO) {
Expand All @@ -135,23 +149,69 @@ class PluginPreview(context: Context) {
} else if (p.content == PortInformation.PORT_CONTENT_TYPE_MIDI) {
if (p.direction == PortInformation.PORT_DIRECTION_INPUT)
midiIn = i
} else if (p.content == PortInformation.PORT_CONTENT_TYPE_MIDI2) {
if (p.direction == PortInformation.PORT_DIRECTION_INPUT)
midi2In = i
}
}

val audioBufferFrameSize = host.audioBufferSizeInBytes / 4 // 4 is sizeof(float)
val controlBufferFrameSize = host.defaultControlBufferSizeInBytes / 4 // 4 is sizeof(float)

(0 until instance.getParameterCount()).map { paraI ->
val para = instance.getParameter(paraI)
// FIXME: implement parameter updates via MIDI port (parameter changes) instead of per-parameter port.
if (midi2In >= 0) {
val midi2Bytes = mutableListOf<Byte>()

(0 until instance.getParameterCount()).map { paraI ->
val para = instance.getParameter(paraI)

val ump = Ump(
UmpFactory.midi2NRPN(
0,
0,
para.id / 0x100,
para.id % 0x100,
parameters[paraI].toRawBits().toLong()
)
)
// generate Assignable Controllers into midi2Bytes.
midi2Bytes.addAll(ump.toPlatformNativeBytes().toTypedArray())
}

val header = MidiHelper.toMidiBufferHeader(0, (parameters.size * 4).toUInt())
midi2Bytes.addAll(0, header)

for (portI in 0 until instance.getPortCount()) {
val port = instance.getPort(portI)
if (para.name == port.name) {
val c = audioProcessingBuffers[portI].order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer()
val localBufferL = audioProcessingBuffers[midi2In]
localBufferL.clear()
localBufferL.put(midi2Bytes.toByteArray(), 0, midi2Bytes.size)
instance.setPortBuffer(midi2In, localBufferL, midi2Bytes.size)
} else {
// If there are parameter elements, look for ports based on each parameter's name.
// If there isn't, just assume parameter index == port index.
if (instance.getParameterCount() > 0) {
(0 until instance.getParameterCount()).map { paraI ->
val para = instance.getParameter(paraI)
for (portI in 0 until instance.getPortCount()) {
if (para.name != instance.getPort(portI).name)
continue
val c = audioProcessingBuffers[portI].order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer()
c.position(0)
// just put one float value
c.put(parameters[paraI])
instance.setPortBuffer(
portI,
audioProcessingBuffers[portI],
audioProcessingBufferSizesInBytes[portI]
)
break
}
}
} else {
for (portI in 0 until instance.getPortCount()) {
val c =
audioProcessingBuffers[portI].order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer()
c.position(0)
// just put one float value
c.put(parameters[paraI])
c.put(parameters[portI])
instance.setPortBuffer(
portI,
audioProcessingBuffers[portI],
Expand All @@ -162,6 +222,10 @@ class PluginPreview(context: Context) {
}
}

val midiSequence = MidiHelper.getMidiSequence()
val midi1Events = MidiHelper.splitMidi1Events(midiSequence.toUByteArray())
val midi1EventsGroups = MidiHelper.groupMidi1EventsByTiming(midi1Events).toList()

instance.activate()

var currentFrame = 0
Expand Down
82 changes: 67 additions & 15 deletions include/aap/ext/parameters.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,11 @@ extern "C" {
#endif

// This parameters extension assumes that parameter updates by host are sent to the plugin using
// MIDI2 input port, and parameter change notification back to the host is sent from the plugin
// using MIDI2 output port.
// MIDI2 inputs, and parameter change notification back to the host is sent from the plugin
// using MIDI2 outputs. The primary source for the inputs is a MIDI2 input port, and the destination
// (note that we do not say "primary" here) should be a MIDI2 output port.
//
// ## MIDI2-parameters mapping schemes
//
// The way how parameter changes and notifications are transmitted depends on get_mapping_policy().
// extension function.
Expand All @@ -19,22 +22,70 @@ extern "C" {
// MIDI2 Assignable Controller (ACC) UMP.
// - When it returns `AAP_PARAMETERS_MAPPING_POLICY_ACC2`, it works quite like
// AAP_PARAMETERS_MAPPING_POLICY_ACC, except that for AAP parameter changes and notifications
// the most significant BIT of the ACC MSB is always 1, so that the range of the valid parameter
// index is limited (but almost harmless in practive - who has >32768 parameters?).
// the most significant BIT of the ACC MSB is always 1 (i.e. always negative as int16_t),
// so that the range of the valid parameter index is limited (but almost harmless in practive -
// who has >32768 parameters?).
// For this mode, plugins are responsible to remove tha flag when decoding the ACC UMP message.
// It is useful IF the plugin should receive any MIDI message sent by host.
// - None does not has any premise. Host in general has no idea how to send the parameter changes.
// Some kind of special hosts may be able to control such a plugin (for example, via some
// Universal System Exclusive messages). Or the plugin might want to
// expose all those parameters but not control and/or notify change publicly (all at once?).
//
// The Assignable controller index is limited to 31-bit, and the mst significant *bit* of the MSB
// is always 1 for AAP parameter change message. For any other Assignable Controller (etc.) messages.
// it is up to host and plugin. (We don't want to "map" and therefore harm MIDI inputs like VST3 does.)
// Supporting parameters up to 32768 would be more than reasonable.
// - None does not have any premise. Host in general has no idea how to send the parameter changes.
// Some specific hosts may be able to control such a plugin (for example, via some
// Universal System Exclusive messages, or based on some Profile Configuration).
// Or the plugin might want to expose all those parameters but not control and/or notify
// changes publicly (but all at once, not per parameter?).
//
// They can also be Relative Assignable Controoller, or Assignable Per-Note Controller, but it is
// They can also be Relative Assignable Controller, or Assignable Per-Note Controller, but it is
// up to the plugin if it accepts them or not.
//
// ## UI events
//
// At this state, we decided to not provide its own event queuing entrypoint in this extension.
// Host is responsible to send remote-UI-originated events within its `process()` calls.
//
// It is easier to ask every host developer to implement event input unification i.e. unify the UI
// event inputs into its MIDI2 inputs for `process()`, than asking every plugin developer to do it.
// AAP has two kinds of UIs: local in-plugin-process UI and remote in-host-process UI:
//
// - For remote UI, the UI is a Web UI which dispatches the input events to the host WebView using
// JavaScript interface, and there is (will be) the event dispatching API for that, as well as
// parameter change notification listener API. Then in terms of host-plugin interaction, it is
// totally a matter of `process()`
//
// - For local UI, it is totally within the plugin process, thus it is up to the plugin itself
// to implement how to interpret its own UI events to the MIDI2 input queue.
//
// In either way, host will receive parameter change notifications through the MIDI2 outputs,
// synchronously. For remote UI, the host will have to dispatch the change notifications to the
// Web UI (via the parameter change notification listener API).
//
// This decision is actually tentative and we may introduce additional event queuing function
// that would enhance use of parameters.
//
// ### Some technical background on parameters and UI
//
// In the world of audio plugin formats, there are two kinds of parameter support "extensions" :
//
// - LV2 has strict distinction between DSP and UI, which results in detached dynamic library
// entity (dll/dylib/so) for UI from DSP, and it needs certain interface (API) for those two.
// Since it is for UI, it is not a synchronous API. It writes events to the port, without
// reading any "response" notifications. They will be sent to output ports at some stage.
// - CLAP has parameters extension in totally different way. It has `flush()` that host can tell
// plugin to process parameters and get parameter change notifications synchronously.
// It does not care about UI/DSP separation. It only cares the interface between host and plugin.
//
// <del>
// The in-process UI and any other out-process UIs, apart from the primary sequencer (DAW) MIDI2
// inputs, should also be able to send parameter changes. That is why we will have `addEvents()`
// member function as part of this extension.
// Note that it is not usable for "receiving" change notifications, as change notifications
// would be often sent as to notify the "latest value" which cannot be really calculated without
// the actual audio processing inputs.
// To not interrupt realtime audio processing loop, additional events from UI should be enqueued,
// not processed within the call to the processing function. We could name the function as
// `process()` instead of `addEvents()` and specify outputs stream as an argument too, but that
// implies we generate the outputs *on time*, which is not realistic (the function should behave
// like "enqueue and return immediately", nothing to process further to get outputs).
// </del>
//

#define AAP_PARAMETERS_EXTENSION_URI "urn://androidaudioplugin.org/extensions/parameters/v1"
#define AAP_PARAMETERS_XMLNS_URI "urn://androidaudioplugin.org/extensions/parameters"
Expand All @@ -52,7 +103,7 @@ typedef struct aap_parameter_info_t {
int32_t stable_id;
// A name string to display, for human beings etc.
char display_name[AAP_MAX_PARAMETER_NAME_CHARS];
// A file-tree style structural path for the parameter.
// A file-tree style structural path for the parameter (following the CLAP way here).
// Applications would split it by '/', ignoring empty entries e.g.:
// `<parameter id="0" name="OSC1 Reverb Send Level", path="/OSC1/Reverb" />`
// `<parameter id="1" name="OSC1 Reverb Send Depth", path="/OSC1/Reverb" />`
Expand All @@ -70,6 +121,7 @@ typedef struct aap_parameter_info_t {
double min_value;
double max_value;
double default_value;
bool per_note_enabled;
} aap_parameter_info_t;

enum aap_parameters_mapping_policy {
Expand Down
Loading

0 comments on commit e6eab86

Please sign in to comment.