SMS Spectral Modeling Synthesis

resynthesis = SMS.ar(input,maxpeaks=80, currentpeaks=80, tolerance=4, noisefloor= 0.2, freqmult=1.0, freqadd=0.0, formantpreserve=0, useifft=0, ampmult=1.0, graphicsbufnum, mul=1.0, add=0.0)

An implementation of the sines+noise model first described by Xavier Serra in his 1989 PhD thesis; an input sound is analysed in terms of sinusoidal components by a peak tracking phase vocoder. The error between the sinusoidal reconstruction and the original signal (the residual) is then modeled by a noise model of filtered white noise. The sines part and the noise part are separately resynthesised, allowing independent transformations.

For technical details see:

Xavier Serra and Julius O. Smith (1990) "Spectral Modeling Synthesis: A Sound Analysis/Synthesis System Based on a Deterministic plus Stochastic Decomposition". Computer Music Journal 14(4): 12--24

Note that this plugin assumes block size of 64 and is optimised for 44100 sampling rate (internally, it uses 1024 point FFTs).

input- Audio rate input to be analysed

maxpeaks- Absolute maximum number of allowed peaks to be detected in the spectrum

currentpeaks- Current number of allowed peaks to be detected in the spectrum

tolerance- Search area for matching peaks; within tolerance spectral bins

noisefloor- Minimum magnitude for a candidate peak (measured as spectral magnitude)

freqmult- Resynthesis parameter to change frequency; currently causes a gross multiplication of frequency of all sinusoidal components

freqadd- Resynthesis parameter to change frequency; currently causes a gross addition of a frequency to all sinusoidal components

formantpreserve- Even if changing the frequencies of sinusoidal partial tracks, re-impose the original magnitude spectrum so as to keep the formants (spectral envelope preservation). 0 is off, otherwise on (there is a small performance hit).

useifft- Use IFFT based resynthesis, which is lower quality, but substantially more efficient

ampmult- amplitude multiplier for internal compensation for window power loss within algorithm. Usually leave as default of 1.0.

graphicsbufnum- Will fill a user provided buffer with sines + noise data; the buffer must be size 1 + 513 + 5*(maxsines). The first entry will be the number of sines active for that polled frame. Default for this argument is -1, meaning do not write any status data. See the example with live plotting at the base of this help file.

//sine reconstruction left channel, noises on right

(

{

var in, fft, output;

in= SoundIn.ar(0);

output=SMS.ar(in, 50,MouseY.kr(1,50), 8, 0.3); //or freqmult: MouseX.kr(0.5,4)

output

}.play

)

//sum reconstruction channels to simulate effect of straight through

(

{

var in, fft, output;

in= SoundIn.ar(0);

output=SMS.ar(in, 50,MouseY.kr(1,50), 8, 0.3); //or freqmult: MouseX.kr(0.5,4)

output.sum.dup

}.play

)

//just noise residual

d=Buffer.read(s,"sounds/a11wlk01.wav");

(

{

var in, fft, output;

in=PlayBuf.ar(1,d,BufRateScale.kr(d),1,0,1);

output=SMS.ar(in, 50,50, MouseX.kr(1.0,10.0).round(1.0),MouseY.kr(0.1,20.0,'exponential'), 1.0);

Out.ar(0,Pan2.ar(output[1]));

}.play

)

//frequency multiplication and shift with formant preservation

(

{

var in, fft, output;

in=PlayBuf.ar(1,d,BufRateScale.kr(d),1,0,1);

output=SMS.ar(in, 60,60, 4.0,0.2, MouseX.kr(0.5,4), MouseY.kr(0,1000), 1.0);

Out.ar(0,Pan2.ar(output[0]));

}.play

)

//transient detection via Pitch hasFreq output

//could have freq input and transient detection input to SMS to control rendering

(

{

var in, fft, sines, noise, freq, hasFreq;

//in= SoundIn.ar(0);

in=PlayBuf.ar(1,d,BufRateScale.kr(d),1,0,1);

#freq, hasFreq= Pitch.kr(in);

#sines, noise=SMS.ar(in, 50,50, 8, 1.0, MouseX.kr(0.5,4));

Pan2.ar(sines*(hasFreq.lag(0.01,0.01)) + LPF.ar(noise,MouseY.kr(100,10000,'exponential')),0.0)

}.play

)

//alternative; only pass into SMS if not a transient region

(

{

var in, fft, sines, noise, freq, hasFreq;

//in= SoundIn.ar(0);

in=PlayBuf.ar(1,d,BufRateScale.kr(d),1,0,1);

#freq, hasFreq= Pitch.kr(in);

#sines, noise=SMS.ar(if(hasFreq,in, Silent.ar), 50,50, 8, 1.0, MouseX.kr(0.5,4));

if(hasFreq,Pan2.ar(sines + LPF.ar(noise,MouseY.kr(100,10000,'exponential')), 0.0),0.25*in)

}.play

)

//having fun

(

{

var in, fft, output;

in=PlayBuf.ar(1,d,BufRateScale.kr(d),1,0,1);

output=SMS.ar(in, 60,60, 4.0,0.2, MouseX.kr(0.5,4), MouseY.kr(0.0001,10000,'exponential'), LFNoise0.kr(2,0.5,0.5).round(1));

Out.ar(0,Pan2.ar(AllpassC.ar(output[0],0.05,LFNoise2.kr(10,0.02,0.025),0.5) + CombL.ar(output[1],0.1,0.1,2),0.0));

}.play

)

//having even more fun

(

{

var in, fft, output;

in=SoundIn.ar(0);

output=SMS.ar(in, 60,60, 4.0,0.1, MouseX.kr(0.01,10), MouseY.kr(-1000,1000), 0.0);

Out.ar(0,Pan2.ar(CombL.ar(output[1],0.02,0.02,0.5) + output[0]));

}.play

)

//testing IFFT resynthesis

(

{

var in, fft, output;

in=PlayBuf.ar(1,d,BufRateScale.kr(d),1,0,1);

output=SMS.ar(in, 300,MouseY.kr(1,300), 10.0,0.01, MouseX.kr(0.5,4), 0, 1.0, 1);

Out.ar(0,output);

}.play

)

//just sines

(

{

var in, fft, output;

in=PlayBuf.ar(1,d,BufRateScale.kr(d),1,0,1);

output=SMS.ar(in, 500,500, MouseX.kr(1.0,10.0).round(1.0),MouseY.kr(0.001,20.0,'exponential'), 1.0, 1.0,0,1);

Out.ar(0,Pan2.ar(output[0]));

}.play

)

//having even more fun, this time with IFFT

(

{

var in, fft, output;

in=SoundIn.ar(0);

output=SMS.ar(in, 200,200, 4.0,0.1, MouseX.kr(0.01,10), MouseY.kr(-1000,1000), 0.0, useifft:1);

Out.ar(0,Pan2.ar(CombL.ar(output[1],0.02,0.02,0.5) + output[0]));

}.play

)

//experimenting with ampmult

(

{

var in, fft, output;

in=SoundIn.ar(0);

output=SMS.ar(in, 200,200, 4.0,0.01,MouseY.kr(0.01,100,'exponential'),0, formantpreserve:1.0, useifft:1, ampmult:MouseX.kr(0.0,10.0));

Out.ar(0,output);

}.play

)

/////////////////////////////////////////////////////////////

//drawing out sine trails

b.free

//buffer for getting data back from UGen

b=Buffer.alloc(s,100*5*2+514,1)

//sine reconstruction left channel, noises on right

(

{

var in, fft, output;

in= SoundIn.ar(0);

output=SMS.ar(in, 100,100, 8, 0.05, useifft:1, graphicsbufnum: b.bufnum); //or freqmult: MouseX.kr(0.5,4)

output

}.play

)

//get data back to language

b.getn(0,1514,{|val| Post << val <<< nl; })

//

//frequency is (bin number/512.0) * pi

//1500*172 would be 258000 floats per second passed!

//plot sines (512) only (not plotting noise for now but could do as (128 bands))

(

var data, basedata, updatedata, paintdata;

var xperframe= 3; //5 values

var fps= 172; //172;

var drawaccum=8; //to avoid too much redraw, accumulate this number of FFT frames worth of data, then plot; 8 is good compromise

var updatesperwindow= 40; //22;

var frameperwindow = drawaccum*updatesperwindow; //176//must be divisible by drawaccum //fps = about 172 = 44100/256

var windowx = xperframe*frameperwindow;

var windowy=640;

var freqconvert= (512.0/pi); //(44100/1024.0)* // unit->m_nover2/pi

var freq1, freq2, amp1, amp2, freq, amp;

var temp;

var black, white;

var counter=0; //for scrolling around

var userview;

var drawdataready=false, drawpos=0;

var i, t;

//var drawupdate = drawaccum/

var bufferfetchwait = (1.0/172.0);

var drawupdatex = drawaccum*xperframe;

black= Integer.fromColor(Color.black);

white= Integer.fromColor(Color.white);

data= Array.fill(windowx*640);

basedata= Int32Array.fill(drawupdatex*windowy,{black});

updatedata=basedata.copy;

i = SCImage.new(windowx@windowy);

w = SCWindow.new("SCImage", Rect(0, 100, windowx+10, windowy+10)).front;

w.onClose_({ t.stop; i.free; }); // free the image when the window is closed

userview = SCUserView(w, w.view.bounds)

.relativeOrigin_(false)

//.focusColor_(Color.white.alpha_(0))

//.resize_(5)

.backgroundImage_(i) //, 10

.drawFunc_({arg view;

//image.drawInRect(view.bounds, image.bounds, 2, 1.0);

//userview.refreshInRect(Rect(10+(counter*drawupdatex),10,drawupdatex,windowy));

i.drawInRect(Rect(10+(counter*xperframe),10,drawupdatex,windowy),Rect(counter*xperframe,0,drawupdatex,windowy), 2, 1.0);

});

t= {

inf.do {

b.getn(0,1514,{|val|

var number = val[0].round(1.0).asInteger;

var drawshift= drawpos*xperframe;

//number.postln;

//updatedata=basedata.copy;

if(drawpos<drawaccum, {

number.do{|i|

var pos= 1+(5*i); //5 values

//freq1, freq2, amp1, amp2, phase

freq1 = (freqconvert*val[pos]); //.round(1.0).asInteger.min(511);

freq2 = (freqconvert*val[pos+1]); //.round(1.0).asInteger.min(511);

//freq1 = freqconvert*val[pos];

freq= (freq1+freq2)*0.5;

//warping

//freq= ((((freq1+freq2)*0.5)/511.0)**0.25)*511.0;

//[\freq1, freq1, \original, val[pos]].postln;

freq= 511-((freq).round(1.0).asInteger.min(511));

amp= (((val[pos+2] + val[pos+3])*0.5)*1024).min(1.0);

//[\freq, freq, \amp, amp].postln;

//goes across by rows then columns down page

//i*windowy+freq1

//(i*windowy+freq1).post; " ".post;

//Integer.fromColor(Color.white);

//was = white

xperframe.do{|i| updatedata[drawshift+i+(freq*drawupdatex)]= Integer.fromColor(Color.green(amp));};

//" ".postln;

};

drawpos=drawpos+1;

if(drawpos==drawaccum,{drawdataready= true;});

});

if(drawdataready,{

drawdataready=false;

paintdata= updatedata.copy;

updatedata = basedata.copy;

drawpos=0;

{

counter=(counter+drawaccum)%frameperwindow;

i.setPixels(

paintdata,

Rect(counter*xperframe,0,drawupdatex,windowy)

);

//drawupdatex

//w.refresh

userview.refreshInRect(Rect(10+(counter*xperframe),10,drawupdatex,windowy));

//i.drawInRect(Rect(10+(counter*xperframe),10,xperframe,windowy), Rect(counter*xperframe,0,xperframe,windowy), 2, 1.0); // only a section

}.defer;

});

});

bufferfetchwait.wait;

}

}.fork;

)