PixelPlayer

PixelPlayer is an interactive environment for exploring image to sound mapping.

In the implementation below, the brightest pixel and darkest pixel in a video feed are mapped to MIDI note pitch (vertical axis) and stereo placement (horizontal axis). The brightest pixel is indicated with a green circle, the darkest with a red circle. New notes are triggered whenever there is a state change. The sound thus reflects the amount of activity in the frame as well as the location of activity.

PixelPlayer uses The MidiBus library, which works on Windows and (we hear) on OS X, but not on Linux.

Pitch mapping is specified in the PITCH_TABLE array. The mapping below reflects a C-major scale; feel free to try different mappings. You will probably have to tweak CAM_WIDTH, CAM_HEIGHT, CAM_FPS, and the output MIDI device (currently hardcoded as “Microsoft MIDI Mapper”) to reflect resources that are available on your machine.

play_two_pixels_change_tonal.pde

/** Reduce resolution of captured image and indicate brightest and darkest
 * pixels with sound. 
 * @author Mithat Konar
 */
 
import processing.video.*;
import themidibus.*;
 
// === Program constants === //
// Rendering parameters:
// number of cells the rendered image should be in each direction:
final int REDUCED_WIDTH = 32;
final int REDUCED_HEIGHT = 24;
 
// Canvas parameters:
// number of times you want REDUCED image blown up:
final int OUTPUT_SCALE = 20;
// frame rate of rendered output:
final int CANVAS_FPS = 2;  // should divide evenly into CAM_FPS
                           // to avoid jitter.
 
// Video capture parameters
// (adjust as neeeded for your platform's available capture options):
final int CAM_WIDTH = 320;
final int CAM_HEIGHT = 240;
final int CAM_FPS = 35;
 
// PITCH_TABLE maps the row number to a MIDI note.
// You'll need  >= REDUCED_HEIGHT values, 0th row is on bottom for now.
final int[] PITCH_TABLE = {
  48, 50, 52, 53, 55, 57, 59, 
  60, 62, 64, 65, 67, 69, 71,
  72, 74, 76, 77, 79, 81, 83,
  84, 86, 88
};
 
// Misc. constants
final int PAN_CONTROLLER = 10;  // MIDI controller number for pan control
final int MIN_ALPHA = 48;       // minimum alpha for drawing pixel highlights
 
// === Global variables ===//
Capture cam;        // The video capture device.
PImage img;         // Buffer image.
MidiBus midiOutDev; // Midi output device
 
int channel;        // MIDI output channel 
int velocity;       // MIDI output velocity
int brightPitch;    // MIDI bright note output pitch 
int brightPan;      // MIDI bright note output pan
int darkPitch;      // MIDI dark note output pitch 
int darkPan;        // MIDI dark note output pan
 
float brightAlpha;  // alpha with which brightest pixel is shown. 
float darkAlpha;    // alpha with which darkest pixel is shown.
 
// === GO! === //
void setup() {
  frameRate(CANVAS_FPS);
  size(REDUCED_WIDTH*OUTPUT_SCALE, REDUCED_HEIGHT*OUTPUT_SCALE);
 
  if (Capture.list().length == 0) {
    println("There are no cameras available for capture.");
    exit();
  }
 
  // Instantiate a buffer image used for subsampling, etc.
  img = createImage(REDUCED_WIDTH, REDUCED_HEIGHT, RGB);
 
  // Instantiate a new Capture object, requesting the specs:
  cam = new Capture(this, CAM_WIDTH, CAM_HEIGHT, CAM_FPS);
  cam.start();  // In Processing 2.0, you need to start the capture device
 
  // Instantiate a MidiBus to provide sound output
  midiOutDev = new MidiBus(this, -1, "Microsoft MIDI Mapper");
 
  // Set MIDI output defaults, might be changed in draw():
  channel = 0;      // MIDI channel number
  velocity = 127;   // MIDI output velocity value
  brightPitch = 48; // MIDI note number 
  brightPan = 64;   // MIDI pan value
  darkPitch = 48;   // MIDI note number 
  darkPan = 64;     // MIDI pan value
 
  // Set some rendering defaults:
  ellipseMode(CORNER);
  brightAlpha = 255.0;
  darkAlpha = 255.0;
 
  // set channel volume (controller #7):
  midiOutDev.sendControllerChange(channel, 7, 127);
}
 
void draw() {
  int brightestCol = 0;
  int brightestRow = 0;
  float bightestIntensity = 0.0;
 
  int darkestCol = 0;
  int darkestRow = 0;
  float darkestIntensity = 255.0; 
 
  int pitch; // a temp var
  int pan;   // a temp var
 
  // Grab a frame
  if (cam.available() == true) {
    cam.read();
  }
 
  // We are using a buffer img because
  // cam.resize(REDUCED_WIDTH, REDUCED_HEIGHT);
  // doesn't work :-(
  img.copy(cam, 0, 0, CAM_WIDTH, CAM_HEIGHT, 0, 0, REDUCED_WIDTH, REDUCED_HEIGHT);
  img.loadPixels();
 
  // For each column in img:
  for (int col = 0; col < REDUCED_WIDTH; col++) {
    // For each row in img:
    for (int row = 0; row < REDUCED_HEIGHT; row++) {
      // Draw the pixel intensity.
      float pixelIntensity = brightness(img.pixels[col + row*img.width]);
      fill(pixelIntensity);
      stroke(pixelIntensity);
      rect(col*OUTPUT_SCALE, row*OUTPUT_SCALE, OUTPUT_SCALE, OUTPUT_SCALE);
 
      // Determine whether this is the brightest pixel in this frame.
      if (pixelIntensity > bightestIntensity)
      {
        bightestIntensity = pixelIntensity;
        brightestCol = col;
        brightestRow = row;
      }
      // Determine whether this is the darkest pixel in this frame.
      else if (pixelIntensity < darkestIntensity)
      {
        darkestIntensity = pixelIntensity;
        darkestCol = col;
        darkestRow = row;
      }
    }
  }
 
  strokeWeight(2);
  fill(#000000, 0);
 
  // Manage notes //
  // brightest pixel:
  pitch = mapNoteNumber(brightestRow);
  pan = mapPan(brightestCol);
  if (pitch != brightPitch || pan != brightPan) {
    midiOutDev.sendNoteOff(channel, brightPitch, velocity);
    brightPitch = pitch;
    brightPan = pan;
    playNote(midiOutDev, channel, brightPitch, velocity, brightPan);
    brightAlpha = 255.0;
  }
  else {
    brightAlpha /= 1.62;
    brightAlpha = brightAlpha < MIN_ALPHA ? MIN_ALPHA : brightAlpha;
  }
 
  stroke(#00ff00, brightAlpha);
  ellipse(brightestCol*OUTPUT_SCALE, brightestRow*OUTPUT_SCALE, 
          OUTPUT_SCALE, OUTPUT_SCALE); // draw pixel highlight
 
  // darkest pixel
  pitch = mapNoteNumber(darkestRow);
  pan = mapPan(darkestCol);
  if (pitch != darkPitch || pan != darkPan) {
    midiOutDev.sendNoteOff(channel, darkPitch, velocity);
    darkPitch = pitch;
    darkPan = pan;
    playNote(midiOutDev, channel, darkPitch, velocity, darkPan);
    darkAlpha = 255;
  }
  else {
    darkAlpha /= 1.62;
    darkAlpha = darkAlpha < MIN_ALPHA ? MIN_ALPHA : darkAlpha;
  }
 
  stroke(#ff0000, darkAlpha);
  ellipse(darkestCol*OUTPUT_SCALE, darkestRow*OUTPUT_SCALE, 
          OUTPUT_SCALE, OUTPUT_SCALE); // draw pixel highlight
}
 
/* Helper Functions */
 
/**
 * Play a note.
 *
 * @param  outDev   output MIDI device (MidiBus)
 * @param  channel  MIDI channel (int)
 * @param  pitch    MIDI note number (int)
 * @param  velocity MIDI velocity (int)
 * @param  pan      MIDI pan value (int)
 * @return void
 */
void playNote(MidiBus outDev, int channel, int pitch, int velocity, int pan) {
  outDev.sendControllerChange(channel, PAN_CONTROLLER, pan);
  outDev.sendNoteOn(channel, pitch, velocity);
}
 
/**
 * Map a row number to a MIDI note number.
 * 
 * @param  rowNum  The row number (int)
 * @return the MIDI note number (int)
 */
int mapNoteNumber(int rowNum) {
//  return 64-rowNum;
  return PITCH_TABLE[REDUCED_HEIGHT-1-rowNum]-6;
}
 
/**
 * Map a column number to a MIDI pan value.
 *
 * @param  colNum  The column number (int)
 * @return the MIDI pan value (int).
 */
int mapPan(int colNum) {
  return round(map(colNum, 0, REDUCED_WIDTH-1, 0, 127));
}