What I needed was a way to grep multiple logs for an exception. But doing "tail -f *.log | grep exception" only let me see the exception, but I didn't know which log file to look in

I have found this guide, and altered the script and added

1. Printing the file name in the beginning of each line
2. Padding spaces after the file name (you can alter the minimal length in the ALIGN variable

Anyway - here's the script. Hope you find it useful

#!/bin/bash
 
# When this exits, exit all back ground process also.
trap 'kill $(jobs -p)' EXIT
 
ALIGN=31
 
# iterate through the each given file names,
for file in "$@"
do
        LENGTH=`echo ${file} | wc -c`
        PADDING=`expr ${ALIGN} - ${LENGTH}`
        PAD=`perl -e "print ' ' x $PADDING;"`
 
        # show tails of each in background.
        tail -f $file | sed "s/^/${file}${PAD}:/g"  &
done
 
# wait .. until CTRL+C
wait

I used C329 with Teensy 2.0, and connected the MISO MOSI SCLK pins to the defaults, the only thing left is to connect the SPI CS pin on the camera module (check out Page 3 in the manual) to some GPIO, I chose the Teensy's B0 pin.

So the operation of the library is very similar to the example online:

#include "CameraC329.h"
#include <SPI.h>

CameraC329 camera(PIN_B0); //set B0 as the CS pin

void getPicture_callback(uint32_t pictureSize, uint16_t packetSize, uint32_t packetStartPosition, byte* packet)
{
//do something with picture buffer
}

void setup() {
          pinMode(PIN_B0, OUTPUT);
          digitalWrite(PIN_B0,HIGH); //unselect

          SPI.setDataMode(SPI_MODE0);
          SPI.begin();

          delay(2000); //time for the module to load stuff from EEPROM

          if (!camera.sync())
          {
            Serial.println("Sync failed");
            camera.powerOff();
            SPI.end();
            return;
          } else {
            Serial.println("Camera sync");
          }
        
          if (!camera.initialize(CameraC329::BAUD14400, CameraC329::CT_JPEG, CameraC329::PR_160x120, CameraC329::JR_320x240))
          {
            Serial.println("Initialize failed");
            return;
          } else {
            Serial.println("Camera init");
          }
          
          if (!camera.setQuality(CameraC329::QL_BEST))
          {
            Serial.println("Set quality failed");
            return;
          } else {
            Serial.println("Camera quality set");
          }

          if (!camera.getPicture(CameraC329::PT_JPEG_PREVIEW, &getPicture_callback))
          {
            Serial.println("Get Picture Failed");
            return;
          }
}

The major changes to the UART library are in the sendCommand and waitForResponse functions

void CameraC329::sendCommand()
{
  uint8_t i;
//Serial.print("send command ");
  // Big endian
  for (i = 0; i < CMD_SIZE; i++)
  {
//Serial.print(outputCommand[i]); Serial.print(" ");
    digitalWrite(spi_cs_pin,LOW);
    SPI.transfer(outputCommand[i]);
    digitalWrite(spi_cs_pin,HIGH);
  }
//Serial.println();
}

bool CameraC329::waitForResponse(uint32_t timeout, byte buffer[], uint16_t bufferLength)
{
  uint8_t byteCount = 0;
  unsigned long time = millis();

//Serial.print("response: ");
  while (millis() - time <= timeout)
  {
//    while (cameraPort.available() > 0) //hmmm...
    {
      digitalWrite(spi_cs_pin,LOW);
      buffer[byteCount] = SPI.transfer(0x00); //cameraPort.read();
      digitalWrite(spi_cs_pin,HIGH);
      
//Serial.print(buffer[byteCount],DEC); Serial.print(" ");
      byteCount++;

      if (byteCount == bufferLength) {
//Serial.println();
        return true;
      }
    }
  }
//Serial.println();
  if (byteCount > 0)
    return true;

  return false;
}

But I also found the GET PIC command to be very very picky. It really only responds very sporadically, so I added a bunch of retries:

bool CameraC329::getPicture(PictureType pictureType, void (*callback)(uint32_t pictureSize, uint16_t packetSize, uint32_t packetStartPosition, byte* packet))
{
  uint32_t pictureSize = 0;

  reset(RT_STATE);

  //Although ACK is part of the spec, we're really waiting for a DATA response, so let's skip waiting for ACK and just wait for DATA
//  while (!waitForACK(RESPONSE_DELAY, CMD_GETPICTURE) && ack_counter < 100) {
//    delay(10);
//  }

  //try to send the GET PIC command for 10 times before giving up
  uint32_t get_pic_tries = 0;
  while(get_pic_tries++ < 10) {
    setOutputCommand(CMD_GETPICTURE, pictureType, 0, 0, 0);
    sendCommand();

    uint32_t max_tries = 0, total_tries = 10;
    while (!(waitForResponse(RESPONSE_DELAY) && inputCommand[3] == CMD_DATA) && max_tries++ < total_tries) {
      //this is not a DATA response (start of the image data), let's wait try again shortly
      Serial.print("not DATA ("); Serial.print(inputCommand[3]); Serial.println(")"); 
      delay(10);
    }
    
    if(max_tries < total_tries) break;
  }
  if(get_pic_tries >= 10) return false;
  
  pictureSize = inputCommand[7] << 8;
  pictureSize |= inputCommand[6] << 8;
  pictureSize |= inputCommand[5];

  uint32_t bytePosition = 0;
  uint8_t package[DEFAULT_PACKAGE_SIZE];

  while (bytePosition < pictureSize)
  {
    if (!waitForResponse(RESPONSE_DELAY, package, DEFAULT_PACKAGE_SIZE))
      return false;

    callback(pictureSize, min(DEFAULT_PACKAGE_SIZE, pictureSize - bytePosition), bytePosition, package);
    bytePosition += DEFAULT_PACKAGE_SIZE;
  }

  return true;
}

Even with that, it's not 100% stable!
I think there's still problem with the SPI timing...

Code can be grabbed at: http://web.media.mit.edu/~roys/src/CameraC329SPI.zip

Enjoy,
Roy.

class ofxFFMPEGVideoWriter {
    //instance variables...

    bool initialized;

public:
    ofxFFMPEGVideoWriter():oc(NULL),codec(NULL),initialized(false),frame_count(1) {}
    
    /**
     * setup the video writer
     * @param output filename, the codec and format will be determined by it. (e.g. "xxx.mpg" will create an MPEG1 file
     * @param width of the frame
     * @param height of the frame
     **/
    void setup(const char* filename, int width, int height);
    /**
     * add a frame to the video file
     * @param the pixels packed in RGB (24-bit RGBRGBRGB...)
     **/
    void addFrame(const uint8_t* pixels);
    /**
     * close the video file and release all datastructs
     **/
    void close();
    /**
     * is the videowriter initialized?
     **/
    bool isInitialized() const { return initialized; }
};

So you can see the API is as simple as possible, and specialized for writing RGB images (24-bit packed, RGBRGBRGB...)
The setup() function will t

The internals are taken from the FFMPEG API examples: http://ffmpeg.org/doxygen/trunk/examples.html
But it's kind of long to put here, so check out the code.

In OpenFrameworks I used it to capture the screen and save it:

//--------------------------------------------------------------
void testApp::setup(){
    ofEnableSmoothing();
    ofBackground(0);
    
    videoWriter.setup("output.mpg", ofGetViewportWidth(), ofGetViewportHeight());
}

//--------------------------------------------------------------
void testApp::update(){
    if(videoWriter.isInitialized()) {
        ofscreenimg.grabScreen(0, 0, ofGetViewportWidth(), ofGetViewportHeight());
        videoWriter.addFrame((const uint8_t*)(ofscreenimg.getPixels()));
    }
}

//--------------------------------------------------------------
void testApp::draw(){
    ofFill();
    ofSetColor(255, 0, 255);
    ofCircle(mousept.x, mousept.y, 15);
}

//--------------------------------------------------------------
void testApp::mousePressed(int x, int y, int button){
    mousept.x = x; mousept.y = y;
}

//--------------------------------------------------------------
void testApp::mouseDragged(int x, int y, int button){
    mousept.x = x; mousept.y = y;
}

//--------------------------------------------------------------
void testApp::mouseReleased(int x, int y, int button){
    mousept.x = -50; mousept.y = -50;
}

Code

http://web.media.mit.edu/~roys/src/FFMPEGVideoWriter.zip

Enjoy!

Training the HPDM

An HPDM is basically a Point Distribution Model (PDM) that has multiple levels - a hierarchy. It encodes the deformation a shape can go through, usually by means of dimension reduction (in our case a PCA). In the Hierarchical PDM we generate a shape-deformation space by PCA, then discretize that space using clustering (like k-means, but an Expectation-Maximization operation works just as well) to achieve another level in the hierarchy.
To train the HPDM we need (many) samples of curves. I got the samples automatically using a Kinect, blob detection, curve extraction etc., but that wasn't the end of it. The samples needed a lot of clean-up!
The raw samples contain the whole arm, which is not something I'd like to track and also it throws off the alignment of all the curves because the variance is big.

So I clean it up a bit by looking a bit further down the curve. But since we're doing an ad-hoc curve extraction from the blob, simply "capping" the curves at a certain point along the curve creates curves that are very badly aligned. For an ideal alignment we need them to be the same exact length (in points), and we need the points to be distributed along the shape in the same way, e.g. point #120 lands on the tip of the index finger in all curves.

Here are 10 curves with the 100th point marked on them, we can see how most of the 100th points align, but some are not even close.

Since we're building a PDM, we model the variation of a single point in the curve (like the 100th point), so we need the variation to be small. We therefor look for an alignment.
We start by picking an initial "mean shape", which in the first iteration is simply the average curve from all samples. But as we've seen a single point can vary greatly so the initial average shape does not reflect the true mean shape.
We proceed by examining each curve, matching vs. the mean shape, adding points at the start or end of the curve so they would align. Now the "100th point test" yields better results, the 100th points are roughly at the same position.

We proceed by performing PCA and then k-means. In the papers there is a discussion around choosing the $k$ , however I used $k = \frac{E}{d}$ where E is the number of curves in the DB (I had ~1300) and $d = 50$ (I also experimented with higher $d$ s but they do not span the space too well). Here are the found centers:

After k-means-ing, the paper then continues to say that the "subregions" (sometimes called "patches") of the shape deformation space should be local, and thus hopefully linear.
They do not use the samples that were attached to each center (that's how k-means clustering works, attached samples to centers thus clustering the samples in space), but rather they look for nearest neighbors in the shape space, to get an overlapping area between subregions / patches.
Then they perform PCA on each bunch of NNs around each center to get a linear patch.
Here are the patches with slight deformation on the principal axis:

The core parameters to tweak in the HPDM are:

• $k$ the number of patches, which should be derived from another two parmeters:
• $d$ (the divisor), which along with $E$ (the number of samples) determines how many samples you have per patch
• $O$ (the overlap) which determines how many nearest neighbors we're looking for per each patch (usually this number is bigger than $E/d$ and therefore the patches overlap)
• the size of the shape space - the dimension of the first PCA, the number of principal components to keep. I had it at 20
• the nubmer of principal components to keep in the subregions, I had it at 1

Using the HPDM in a Particle Filter

A quick reminder, Particle Filters (PF) are a way to maintain many different hypotheses for the model (e.g. position of an object, shape, signal, etc.) looking for an unknown truth state of the model while receiving noisy measurements.
In our case we're looking at the position, orientation and shape of a 2D hand object, so each particle is going to hold data that will describe such a model.

class Particle {
public:
	//according to H&H section 3.1
	Point2f centroid;
	float scale;
	float orientation;
	Mat_<float> deformation;
	int patch_num;
}

To employ the PF we need to learn something about the shape - how it changes over time. The PF (a stochastic Markovian process) is dependent of the last iteration to determine the next, we therefore need to extract a shape variation model using the HPDM.
So we go back to the samples, this time looking in chronological order for the relationship between the shape in one frame and the next frame. This can give us an idea of how the shape changes through time, and thus can be used as a prior for the PF.
To that end we construct a transition matrix, one that tells us what is the probability of transforming from shape described by patch i to a shape described by patch j. Easily we just go over the curves sequentially and accumulate the transitions. Most of the frame-to-frame transitions are going to remain in the same patch...

In the end we get a transition matrix:

But we also need a cumulative version of it (accumulating per row) that we will use in the particle filter.

Building the particle filter

Particle filters work by assigning a score for each particle on how well it performs in this iteration, so that in later iterations we sample new particles that will partly preserve this strength but also be open for new hypotheses.
To evaluate how good a particle is we examine it's adherence to edges in the image. Since I am working with a binary mask from the Kinect it has very strong and clear edges, but this method works the same way for regular images.

The red line is the hypothesized shape in that particle, the cyan circles mark if an edge was hit along the normal to the curve.
Scores are given based on how many edges were hit, and if they are pointing in the right direction (i.e. the edge is aligned with the curve).

Normalizing weights, resampling

The standard PF part is normalizing weights of the current particles, and create the cumulative sum which is essential for easily choosing a

//sum all weights
Scalar s = sum(unnormalized_weights);
		
//calc normalized weights by dividing, and build cumulative distribution function
for (int i=0; i<unnormalized_weights.size(); i++) {
	normalized_weights[i] = unnormalized_weights[i] / s[0];
	if (i==0) {
		cumulative_sum[i] = normalized_weights[i];
	} else {
		cumulative_sum[i] = cumulative_sum[i-1] + normalized_weights[i];
	}
}

We then resample new particles form the distribution we created with the current bunch of particles and the cumulative sum (this is called Sequential importance resampling):

void resampleParticles() {
	//draw N particles from the new distribution function (the importance density)
	vector<Particle> new_particles(num_particles);
	for (int i=0; i<particles.size(); i++) {
		while (true) {
			float uni_rand = rng_.uniform(0.0f,1.0f); //randomize a number from [0,1]

			//binary search for position in cumulative sum
			vector<float>::iterator pos = lower_bound(cumulative_sum.begin(), cumulative_sum.end(), uni_rand);
			int ipos = distance(cumulative_sum.begin(), pos);
			if(particles[ipos].scale < 0.85) continue; //re-sample if scale is bad
			new_particles[i].scale = particles[ipos].scale;
			new_particles[i].orientation = particles[ipos].orientation;
			new_particles[i].centroid = particles[ipos].centroid;
			new_particles[i].patch_num = particles[ipos].patch_num;
			particles[ipos].deformation.copyTo(new_particles[i].deformation);
			break;
		}
	}
		
	particles.clear(); particles.insert(particles.begin(),new_particles.begin(),new_particles.end());		
}

Local optimization of particles

In their paper Heap&Hogg suggest we combing a few local optimization iterations with each step of the PF, and claim (rightfully) that it results in faster convergence and better results. The meaning of "local" means that we run it on the same frame a number of times, inside one iteration of the PF.
To help the particles attract to the edges in the image, I try to find a rigid transformation that will decrease the energy of attraction to edges. This is a risky operation as it can throw the curve completely out of balance if the initial guess is no good.

The rigid transformation is found by Kabsch algorithm.
We can also consider non-rigid transformations by looking at the patch deformation axes. I found this method not to be of great use, plus they add a ton of computation that hinders performance.

Propagating

To make the PF function we need to "propagate" the error through the particles. Basically meaning we need to put in some randomness in the process or else it quickly converges and gets stuck.
I therefore use the transition matrix to see where each particle-shape likes to go (or, is likely to go). Actually the cumulative transition matrix:

Each row is the cumulative distribution function that tells what is the probablility of transitioning from prototypical-shape (patch) i to prototypical-shape (patch) j. I use that to randomly pick a target prototypical shape for each particle.

void propagate() {
	//according to section 3.2
	int changed = 0;
	for (int i=0; i<particles.size(); i++) {
		float uni_rand = rng_.uniform(0.0f,1.0f);

		//binary search for new subregion (patch) in Cumulative Transition Matrix
		vector<double> row_a_in_CumulativeMat(Cumulative.cols); 
		int old_patch_num = particles[i].patch_num;
		Cumulative.row(old_patch_num).copyTo(row_a_in_CumulativeMat);
		vector<double>::iterator pos = upper_bound(row_a_in_CumulativeMat.begin(), row_a_in_CumulativeMat.end(), uni_rand);
		int new_patch_num = distance(row_a_in_CumulativeMat.begin(), pos);
//		cout << "particle "<<i<<" ("<<uni_rand<<") ["<<particles[i].patch_num<<"] -> ["<<new_patch_num<<"]\n";
		if (new_patch_num != old_patch_num) {
			//moved to a new patch ("leap through wormhole"): take the mean deformation of that patch
			particles[i].patch_num = new_patch_num;
			hpdm.getPatchMean(new_patch_num).copyTo(particles[i].deformation);
			changed++;
		}
			 
		add_noise(particles[i]);
			
		//TODO: particle should not go out of bounds or be too small/big or orientate too much
	}
	cout << changed << " particles changed, " << (num_particles-changed) << " stayed\n";
}

Each particle then is added some random noise so the filter will not converge and actually be able to cope with movement and deformation of the shape.

Getting the best guess of the PF

Simply by looking at all the particles at a given moment, we can take the best ones and average over them.

void calculateMeanShape() {
	Particle p;
        p.scale = p.orientation = p.centroid.x = p.centroid.y = 0;
	p.deformation.create(1,hpdm.getShapeSpaceDim());
	p.deformation.setTo(0);

        //Take the particles that score the highest
        float N_precentile = *(max_element(normalized_weights.begin(),normalized_weights.end())) * 0.95;

	float sum_w = 0.0;
	for (int i=0; i<num_particles; i++) {
		if (normalized_weights[i] < N_precentile) {
			continue;
		}
		p.orientation += particles[i].orientation * normalized_weights[i];
		p.scale += particles[i].scale * normalized_weights[i];
		p.centroid += particles[i].centroid * normalized_weights[i];
		addWeighted(p.deformation, 1.0, particles[i].deformation, normalized_weights[i], 0.0, p.deformation);
		sum_w += normalized_weights[i];
	}
	p.orientation /= sum_w;
	p.scale /= sum_w;
	p.centroid *= 1.0/sum_w;
	p.deformation /= sum_w;

	mean_shape = getShapeForParticle(p);
}

Here is the filter running with all particles showing (40 particles). The yellow shape is the highest ranking shape, the pink shape is the average of the highest ranking shapes.

Code and Usage

You can grab the code with a usage example here: http://web.media.mit.edu/~roys/src/HHParticleFilter.zip

Thanks!
Roy.

1. What is the skin region in a given image? If we don't yet have a model for it. I call that the Bootstrap problem,
2. What colorspace to use/histogram?
3. The per pixel decision parameter (theta in the literature)

Bootstrap

For bostratpping (namely, getting an initial guess for skin pixels) I used the method from Gomez & Morales 2002:

//get HSV
Mat hsv; cvtColor(rgb, hsv, CV_BGR2HSV);
//get Normalized RGB (aka rgb)
Mat nrgb = getNormalizedRGB(rgb);
		
//take the pixels that are inside the ranges in both colorspaces
Mat mask_hsv, mask_nrgb;
//H=[0,50], S= [0.20,0.68] and V= [0.35,1.0]
inRange(hsv, Scalar(0,0.2*255.0,0.35*255.0), Scalar(50.0/2.0,0.68*255.0,1.0*255.0), mask_hsv);
//r = [0.36,0.465], g = [0.28,0.363]
inRange(nrgb, Scalar(0,0.28,0.36), Scalar(1.0,0.363,0.465), mask_nrgb);

//combine the masks
outputmask = mask_hsv & mask_nrgb;

Input RGB

Bootstrap mask

Training

After we get a bootstrap, which you can see is pretty bad, we can start training the model to get a better segmentation. Essentially this means calculating 2 histograms, one for skin pixels and one for non-skin pixels.

Mat nrgb = getNormalizedRGB(img_rgb);

MatND skin_hist,non_skin_hist;
skin_hist = calc_rg_hist(nrgb,mask);
non_skin_hist = calc_rg_hist(nrgb,~mask);
		
//create a probabilty density function
float skin_pixels = countNonZero(mask), non_skin_pixels = countNonZero(~mask);
for (int ubin=0; ubin < 250; ubin++) {
	for (int vbin = 0; vbin < 250; vbin++) {
		if (skin_hist.at<float>(ubin,vbin) > 0) {
			skin_hist.at<float>(ubin,vbin) /= skin_pixels;
		}
		if (non_skin_hist.at<float>(ubin,vbin) > 0) {
			non_skin_hist.at<float>(ubin,vbin) /= non_skin_pixels;
		}
	}
}

We normalize the histograms with the number of skin (and non-skin) pixels to create a Probability Density Function (that sums to 1).

Skin pixels histogram

Non-skin pixels histogram

Predicting

The SPM algorithm simply states you make a binary prediction (yes or no) for each pixel based on a $\theta$ value (a variable given to the algorithm):

The implementation is trivial if you already have the normalized histograms (and thus PDFs):

float skin_hist_val = skin_hist.at<float>(gbin,rbin);
if (skin_hist_val > 0) {
	float non_skin_hist_val = non_skin_hist.at<float>(gbin,rbin);
	if (non_skin_hist_val > 0) {
		if((skin_hist_val / non_skin_hist_val) > theta_thresh)
			result_mask(i) = 255;
		else 
			result_mask(i) = 0;
	} else {
		result_mask(i) = 0;
	}
} else {
	result_mask(i) = 0;
}

Retraining

I find it good to re-train the model a number of time. What I mean is that I iterate a number of times with train-predict operations, where every iteration I use the result of the prediction to train the model again:

for (int i=0; i<10; i++) { //predict-train N times for convergence
      medianBlur(mask, mask, 3);
      spm.train(cameraframe, mask);
      spm.predict(cameraframe, mask);
      imshow("mask", mask);
      waitKey(50);
}

The results get kind of better with each retraining. Of course there's risk of converging around something which is not skin, and then it all goes to hell.

skin region mask after 10 re-trains

input RGB

skin hist after 10 retrains

non skin hist after 10 re-trains

Elliptical Boundary Model

I learned of the elliptical boundary model through the paper by Lee and Yoo from 2002. But I found the results to be lesser than those of the SPM. In fact I anticipated that, since the EBM tries to model the distribution (or rather give a criterion for pixel-based decision based on the distribution), where the SPMs make no modeling of the distribution and use it as is. The SPMs are more memory-intensive (keeping the entire histograms in memory), but are more accurate since all the information is retained.

The EBM model the skin color distribution by fitting an ellips(oid) to the distribution of skin color values in the histogram.

And makes a decision by using the following inequality: $[X-\psi]^t \Lambda^{-1} [X-\psi] > \theta$
Where $X$ is the tested pixel color-value in r-g space, $\psi$ is the mean value in the distribution and $\Lambda$ is calculated like so:
$\Lambda = \frac{1}{N} \sum_{i=1}^n {f_i (X_i - \mu) (X_i - \mu)^t}$
Where $\mu$ is a weighted mean of the colors in the histogram, $f_i$ is the frequency of the color-value $X_i$ - both of which come from the histogram.
So you kind of see how simple it is to calculate it: (given f_hist is the histogram on the r-g space)

	void train() {
		T ustep = range_dist[0]/hist_bins[0], vstep = range_dist[1]/hist_bins[1];
		
		//calc n, X_i and mu
		Mat_<T> mu(1,2); mu.setTo(0);
		vector<T> f;
		int n = countNonZero(f_hist);
        int count = 0;
        int N = 0;
		Mat_<T> X(n,2);
		for (T ubin=0; ubin < hist_bins[0]; ubin++) {
			for (T vbin = 0; vbin < hist_bins[1]; vbin++) {
                T histval = f_hist.at<T>(ubin,vbin);
				if (histval > 0) {
					Mat_<T> sampleX = (Mat_<T>(1,2) << low_range[0] + ustep * (ubin+.5), low_range[1] + vstep * (vbin+.5));
                    sampleX.copyTo(X.row(count++));
					f.push_back(histval);
					mu += histval * sampleX;
                    N += histval;
				}
			}
		}
        
		mu /= (T)N;
						
		//calc psi - mean of DB
		reduce(X, psi, 0, CV_REDUCE_AVG);
		
		//calc Lambda
		Lambda.create(2,2);
		for (int i=0; i < n; i++) {
			Mat_<T> X_m_mu = (X.row(i) - mu);
			Mat_<T> prod = f[i] * X_m_mu.t() * X_m_mu;
			Lambda += prod;
		}
		Lambda /= N;
		Mat_<T> linv = Lambda.inv();
		Lambda_inv.val[0] = linv(0,0);
		Lambda_inv.val[1] = linv(0,1);
		Lambda_inv.val[2] = linv(1,0);
		Lambda_inv.val[3] = linv(1,1);
		
		cout << "n = " << n << " N = " << N << " mu " << mu << "\npsi " << psi << "\nlambda_inv "<< Lambda_inv<<"\n";
	}

But like I said the results are not so good: ( $\theta = 2$ in this case)

There's again the option of "retraining" the EBM, although this time it's basically just accumulating the r-g histogram with more pixels.

Code

Check the code out, with a snippet on how to use it at: http://web.media.mit.edu/~roys/src/SPMandEBM.zip

Thanks!
Roy.

So a friend gave me a locked device. This device had no Google account linked to it (which prevented me from bypassing the lock with that account), there was no root or ADB access via recovery, and the USB debugging option was disabled.

I found this guide here, which states some commands you can type in ADB shell and should deactivate the pattern lock. So – How do I get ADB access on that phone?

My first try was to flash a recovery image which allows ADB access. This allegedly easy task that is normally done with Odin, turned out to be impossible. I managed to download all the necessary files, but Odin (which for those of you who are unfamiliar – is the software you use with Samsung phones to flash partitions, repartition and basically everything regarding flashing) refused to detect the phone. I had on my PC another version of Odin, for other phones, which detected the phone, but I could not use it with the files compatible with this model.

So, back to snooping the forums. I found a flashable zip file (Link here – remember – Only Galaxy Mini) which allowed me to start ClockworkMod recovery on that phone.

Cool – Now I will had adb access and can type in the commands and go to eat my lunch. WRONG. I need I got CWM to start, but my PC did not recognize any ADB interface. I'm not sure why, either this CWM didn't support ADB, or that fact that it just "rebooted" into CWM from the normal recovery didn't allow it to grant the ADB access. My guess was that if I would have been able to flash it properly – it would have worked.

So now I have CWM, no ADB access and commands I'm unable to type.

And then I understood something – What if I could just create a zip file that runs these commands? I was not the first one to think about this.

So.. I tried flashing this zip. Didn't work. Looking at what the zip is doing – I understood what was the problem. The zip tries to mount /system and /data using busybox. Which… I didn't have… since I was using stock unrooted ROM.

Last try – I tried mounting /system and /data via CWM. Did that and Retried flashing that unlock zip. IT WORKED!!

This took a lot of thought and thinking out of the box. But it was worth it.

So, to summarize what I've done to unlock the device

1. Placed the CWM zip and Pattern Unlock zip on the microSD
2. Started the device to stock recovery (When device is off – Middle button + Volume UP + Power
3. Chose "Install update zip" and selected CWM zip. Now CWM interface started up
4. Go to "mounts and storage menu" and mount /data and /system
5. Go back, and flash the pattern unlock zip
6. Reboot phone

This process was exhausting, but worth it. I really hope you will find this post useful

P.S - The photo on the right is wrong, mounting /cache is unnecessary ;) my photographic mistake

Well because privacy is involved, the least I could do is add SSL somehow. I googled a bit and came across this article. I decided to use my raspberry pi for that.

The process itself is relatively easy but I had to do some improvisations over the article above. So I decided to make a tutorial for this.

You can use this to add SSL layer on top of every http you have.

So here we go:

1. Install apache2 on your raspberry pi:

   sudo apt-get install apache2

2. Enable ssl, proxy and proxy_http modules:

   sudo a2enmod ssl
   sudo a2enmod proxy
   sudo a2enmod proxy_http

3. Add listener to port 10001 (You can use any port, this is from the sample)
   I have used ports.conf file to add it. You can create a new configuration file for this if you like

   NameVirtualHost *:10001
   
   Listen 10001

4. Add definitions for you camera's reverse proxy (VirtualHost tag)

   Few notes:

• The camera's internal address and port in this example is 192.168.9.11:81
• The logs are located in /var/log/apache2 - you can change it to a path you desire
• The certificate are self signed. It is explained next

<VirtualHost *:10001>
 ProxyRequests Off
 ProxyPreserveHost On
 ProxyVia On

 Order deny,allow
 Allow from all

 ProxyPass / http://192.168.9.11:81/
 ProxyPassReverse / http://192.168.9.11:81/
 CustomLog /var/log/apache2/access_cam1.log combined
 ErrorLog /var/log/apache2/error_cam1.log
 ServerName cam1

 SSLEngine On
 SSLCertificateFile /etc/apache2/mycrt.crt
 SSLCertificateKeyFile /etc/apache2/mycrt.key

 <FilesMatch "\.(cgi|shtml|phtml|php)$">
 SSLOptions +StdEnvVars

 SSLOptions +StdEnvVars

 BrowserMatch "MSIE [2-6]" \
 nokeepalive ssl-unclean-shutdown \
 downgrade-1.0 force-response-1.0
 # MSIE 7 and newer should be able to use keepalive
 BrowserMatch "MSIE [17-9]" ssl-unclean-shutdown
</VirtualHost>

Now we need to create certificate files. I used this guide in order to do it. It's pretty straight forward (In my example above I've called them "mycrt")

We're almost good to go. In case you created a passphrase for you certificate, you will notice that as you restart your apache2 it will require you to enter your passphrase.

This can be avoided, if you'd like.

• Create a shell script that echoes your passphrase:
  file: nopass.sh

  #!/bin/bash
   echo your-password

• Add execute permissions to this script
• Edit ssl.conf, and change the SSLPassPhraseDialog line

  SSLPassPhraseDialog exec:/etc/apache2/nopass.sh

That's it! Now you can restart your apahce with

sudo service apahce2 restart

Now try accessing your raspberry pi with https://<ip>:10001

You should see your webcam's login interface with SSL

Feel free to port-forward from your router to your raspberry pi's new SSL port in order to access your camera from the outer world

I hope this guide is useful for you.

I wanna share some code I've been working on lately that implements smooth shape manipulation using Moving Least Squares. More specifically, the excellent simple and powerful method by Schaefer et al. from Texas A&M and Rice universities (great paper). The method was written for image deformation but very straightforward modifications made it work perfectly for 2D shapes and open curves.

Let's get down to business

Affine, similarity and rigid deformations

The beauty with Schefer's method is that it simple to implement, presents powerful results and is blazing-fast because of the optimizations they suggest in the paper (decomposing the math to allow pre-computing as much as possible). Initially they show three methods for deformation: Affine, Similarity and Rigid.
Affine deformation means that, locally, affected points could undergo a complete affine transformation, meaning: Rotation, Translation, non uniform Scale, Shear and Squeeze. While it is still cool, it can unrealistically distort the shapes.

See how in extreme stretch the top fin and mouth of the fish are distorted.

Similarity transformations allow only Rotation, Translation and uniform Scale, which makes the deformation more appealing, but it does tend to enlarge parts on extreme stretch:

Finally, Rigid transformation only allow local Rotation and Translation, keeping the deformed shape more rigid:

Using the code

Everything is encapsulated in the SchaeferMLS template class:

template<typename T>
class SchaeferMLS {
public:
	void Init(const vector<Point_<T> >& curve, const vector<int>& control_idx) {...}
	
	void UpdateAffine() {...}
	void UpdateSimilarity() {...}
	void UpdateRigid() {...}
	
	const vector<Point_<T> >& GetControlPts() { ...}
	vector<Point_<T> >& GetDeformedControlPts() { ...}
	
	void Draw(Mat& img) {...}
};

I've written a small app to show simple usage with the different deformation types, it's fairly simple to follow.
But the gist is:

//Read curve
vector<Point> a;
GetCurveForImage(imread("a_slihouette.png", a, false);

//Convert to Point_<double> - optional
vector<Point2d> a_p2d, a_p2d_smoothed;
ConvertCurve(a, a_p2d);

//Get curvature extrema points
vector<pair<char,int> > stringrep = CurvatureExtrema(a_p2d, a_p2d_smoothed,0.05,4.0);

//Get extrema as control points
vector<int> control_pts; 
for(int i=0;i<stringrep.size();i++) {
	control_pts.push_back(stringrep[i].second);
}

smls.Init(a_p2d, control_pts);
smls.UpdateRigid();
	
Mat visualized_curve(500,500,CV_8UC3);
smls.Draw(visualized_curve);

namedWindow("MLS");
//Implement the onMouse function (or take from repo) to reflect changes in control points
setMouseCallback("MLS", onMouse, NULL);
imshow("MLS", visualized_curve);
waitKey();

You can grab the code at github: https://github.com/royshil/CurveDeformationMLS

Thanks for tuning in!
Roy.

Principle

The idea is simple, take the curve and build a "Signature Database" for it, which then you compare to another DB.
The database is essentially every possible sub-curve of the curve, under every possible offset, all normalized to the same length.
This also makes this method quite robust to occlusions! (where only part of the target curve is available)

The matching is done over the curvature of (a smoothed version of) the curve, the 2nd derivative, which was nicely explained in this paper by Mokhtarian in 2002.

Working with the functions

I created a sample app that takes in a curve (binary image with contour extracted), transforms it and then tries to find a match with the original.
But the central function is

template
void CompareCurvesUsingSignatureDB(const vector<Point_ >& a,
								   const vector<Point_ >& b,
								   int& a_len,
								   int& a_off,
								   int& b_len,
								   int& b_off,
								   double& score
								   );

It takes in any type of curve (Point, Point2f, Point2d) and returns the length and offset inside each curve of the sub-curves that best match.
You can then find a rigid transformation easily by

//assume you have vector a,b; from before

//Get matched subsets of curves
vector a_subset(a.begin() + a_off, a.begin() + a_off + a_len);
vector b_subset(b.begin() + b_off, b.begin() + b_off + b_len);

//Normalize to equal length
ResampleCurve(a_subset, a_subset, 200, true);
ResampleCurve(b_subset, b_subset, 200, true);

//Find rigid transformation
Mat trans = Find2DRigidTransform(a_subset, b_subset);
vector a_trans;
cv::transform(a_subset,a_trans,trans);

Code and salutations

Get the code at the github repo

Enjoy!
Roy.

Smooth operator

So I've been implementing a paper called "Affine-invariant Shape Matching and Recognition Under Partial
Occlusion" by Mai, Chang and Hung from 2010 (Here), that is using the CSS Image (Curvature Scale Space, rather an old concept, developed by Mukhtarian in the mid-1990s) to extract affine-invariant points on curves. The work is rather simple and the paper is short, but nevertheless powerful.
In the process of computing the CSS Image, once must smooth the curve with gaussian kernels of varying sizes, and this I'd like to show you.

The implementation is simple:

/* 1st and 2nd derivative of 1D gaussian 
 */
void getGaussianDerivs(double sigma, int M, vector<double>& gaussian, vector<double>& dg, vector<double>& d2g) {
	int L = (M - 1) / 2;
	double sigma_sq = sigma * sigma;
	double sigma_quad = sigma_sq*sigma_sq;
	dg.resize(M); d2g.resize(M); gaussian.resize(M);
	
	Mat_<double> g = getGaussianKernel(M, sigma, CV_64F);
	for (double i = -L; i < L+1.0; i += 1.0) {
		int idx = (int)(i+L);
		gaussian[idx] = g(idx);
		// from http://www.cedar.buffalo.edu/~srihari/CSE555/Normal2.pdf
		dg[idx] = (-i/sigma_sq) * g(idx);
		d2g[idx] = (-sigma_sq + i*i)/sigma_quad * g(idx);
	}
}

/* 1st and 2nd derivative of smoothed curve point */
void getdX(vector<double> x, 
		   int n, 
		   double sigma, 
		   double& gx, 
		   double& dgx, 
		   double& d2gx, 
		   vector<double> g, 
		   vector<double> dg, 
		   vector<double> d2g,
		   bool isOpen = false) 
{		
	int L = (g.size() - 1) / 2;

	gx = dgx = d2gx = 0.0;
//	cout << "Point " << n << ": ";
	for (int k = -L; k < L+1; k++) {
		double x_n_k;
		if (n-k < 0) {
			if (isOpen) {
				//open curve - mirror values on border
				x_n_k = x[-(n-k)]; 
			} else {
				//closed curve - take values from end of curve
				x_n_k = x[x.size()+(n-k)];
			}			
		} else if(n-k > x.size()-1) {
			if (isOpen) {
				//mirror value on border
				x_n_k = x[n+k]; 
			} else {
				x_n_k = x[(n-k)-(x.size())];
			}			
		} else {
//			cout << n-k;
			x_n_k = x[n-k];
		}
//		cout << "* g[" << g[k + L] << "], ";

		gx += x_n_k * g[k + L]; //gaussians go [0 -> M-1]
		dgx += x_n_k * dg[k + L]; 
		d2gx += x_n_k * d2g[k + L];
	}
//	cout << endl;
}


/* 0th, 1st and 2nd derivatives of whole smoothed curve */
void getdXcurve(vector<double> x, 
				double sigma, 
				vector<double>& gx, 
				vector<double>& dx, 
				vector<double>& d2x, 
				vector<double> g, 
				vector<double> dg, 
				vector<double> d2g,
				bool isOpen = false) 
{	
	gx.resize(x.size()); 
	dx.resize(x.size()); 
	d2x.resize(x.size());
	for (int i=0; i<x.size(); i++) {
		double gausx,dgx,d2gx; getdX(x,i,sigma,gausx,dgx,d2gx,g,dg,d2g,isOpen);
		gx[i] = gausx;
		dx[i] = dgx;
		d2x[i] = d2gx;
	}
}

The smoothing runs on each of the curves dimensions (i.e. x and y) separately, so I create little helper functions:

template<typename T, typename V>
void PolyLineSplit(const vector<Point_<T> >& pl,vector<V>& contourx, vector<V>& contoury) {
	contourx.resize(pl.size()); 
	contoury.resize(pl.size());
	
	for (int j=0; j<pl.size(); j++) 
	{ 
		contourx[j] = (V)(pl[j].x); 
		contoury[j] = (V)(pl[j].y); 
	}
}

template<typename T, typename V>
void PolyLineMerge(vector<Point_<T> >& pl, const vector<V>& contourx, const vector<V>& contoury) {
	assert(contourx.size()==contoury.size());
	pl.resize(contourx.size());
	for (int j=0; j<contourx.size(); j++) {
		pl[j].x = (T)(contourx[j]);
		pl[j].y = (T)(contoury[j]);
	}
}

They also convert to different types if needed.

Smoothing is easy:

vector<Point> curve = ...;

double sigma = 3.0;
int M = round((10.0*sigma+1.0) / 2.0) * 2 - 1;
assert(M % 2 == 1); //M is an odd number

//create kernels
vector<double> g,dg,d2g; getGaussianDerivs(sigma,M,g,dg,d2g);
	
vector<double> curvex,curvey,smoothx,smoothy;
PolyLineSplit(curve,curvex,curvey);

vector<double> X,XX,Y,YY;
getdXcurve(curvex,sigma,smoothx,X,XX,g,dg,d2g,isOpen);
getdXcurve(curvey,sigma,smoothy,Y,YY,g,dg,d2g,isOpen);

PolyLineMerge(curve,smoothx,smoothy);

Resampling

I implemented my own resampling code, where I sample points on the curve keeping the same regular distance between them. To get the regular distance I simply take the complete length of the original curve and divide by the number of desired points.

The implementation is again quite simple:

void ResampleCurve(const vector<double>& curvex, const vector<double>& curvey,
				   vector<double>& resampleX, vector<double>& resampleY,
				   int N,
				   bool isOpen
				   ) {
	assert(curvex.size()>0 && curvey.size()>0 && curvex.size()==curvey.size());
	
	vector<Point2d> resamplepl(N); resamplepl[0].x = curvex[0]; resamplepl[0].y = curvey[0];
	vector<Point2i> pl; PolyLineMerge(pl,curvex,curvey);

	double pl_length = arcLength(pl, false);
	double resample_size = pl_length / (double)N;
	int curr = 0;
	double dist = 0.0;
	for (int i=1; i<N; ) {		
		assert(curr < pl.size() - 1);
		double last_dist = norm(pl[curr] - pl[curr+1]);
		dist += last_dist;
//		cout << curr << " and " << curr+1 << "\t\t" << last_dist << " ("<<dist<<")"<<endl;
		if (dist >= resample_size) {
			//put a point on line
			double _d = last_dist - (dist-resample_size);
			Point2d cp(pl[curr].x,pl[curr].y),cp1(pl[curr+1].x,pl[curr+1].y);
			Point2d dirv = cp1-cp; dirv = dirv * (1.0 / norm(dirv));
//			cout << "point " << i << " between " << curr << " and " << curr+1 << " remaining " << dist << endl;
			assert(i < resamplepl.size());
			resamplepl[i] = cp + dirv * _d;
			i++;
			
			dist = last_dist - _d; //remaining dist			
			
			//if remaining dist to next point needs more sampling... (within some epsilon)
			while (dist - resample_size > 1e-3) {
//				cout << "point " << i << " between " << curr << " and " << curr+1 << " remaining " << dist << endl;
				assert(i < resamplepl.size());
				resamplepl[i] = resamplepl[i-1] + dirv * resample_size;
				dist -= resample_size;
				i++;
			}
		}
		
		curr++;
	}
	
	PolyLineSplit(resamplepl,resampleX,resampleY);
}

Essentially I just traverse the original curve, advancing in a regular pace (resample_size) and adding new vertices as long as there is space to so on the current edge. If space to add more vertices on the edge - I move to the next edge and add more vertices there.

Curvature Scale Space

The core idea of the CSS image (as shown perhaps best by Mokhtarian and Abbassi in their 2002 paper) is to smooth the curve by ever increasing gaussian kernels and looking at zero crossings of the 2nd derivative. When you're smoothing the curve you will loose the noise, and be left with only the biggest changes in curvature, and a zero-crossing on the 2nd derivative will indicate a big change in curvature. The invariant points across the different sizes of the gaussian kernel - are the interest points of the curve (sometimes referred to as "affine invariant").

/* compute curvature of curve after gaussian smoothing 
 from "Shape similarity retrieval under affine transforms", Mokhtarian & Abbasi 2002
 curvex - x position of points
 curvey - y position of points
 kappa - curvature coeff for each point
 sigma - gaussian sigma
 */
void ComputeCurveCSS(const vector<double>& curvex, 
					 const vector<double>& curvey, 
					 vector<double>& kappa, 
					 vector<double>& smoothX, vector<double>& smoothY,
					 double sigma,
					 bool isOpen
					 ) 
{
	int M = round((10.0*sigma+1.0) / 2.0) * 2 - 1;
	assert(M % 2 == 1); //M is an odd number
	
	vector<double> g,dg,d2g; getGaussianDerivs(sigma,M,g,dg,d2g);
	
	vector<double> X,XX,Y,YY;
	getdXcurve(curvex,sigma,smoothX,X,XX,g,dg,d2g,isOpen);
	getdXcurve(curvey,sigma,smoothY,Y,YY,g,dg,d2g,isOpen);
	
	kappa.resize(curvex.size());
	for (int i=0; i<curvex.size(); i++) {
		// Mokhtarian 02' eqn (4)
		kappa[i] = (X[i]*YY[i] - XX[i]*Y[i]) / pow(X[i]*X[i] + Y[i]*Y[i], 1.5);
	}
}

/* find zero crossings on curvature */
vector<int> FindCSSInterestPoints(const vector<double>& kappa) {
	vector<int> crossings;
	for (int i=0; i<kappa.size()-1; i++) {
		if ((kappa[i] < 0 && kappa[i+1] > 0) || kappa[i] > 0 && kappa[i+1] < 0) {
			crossings.push_back(i);
		}
	}
	return crossings;
}

The evolution of the curve (iterating through many levels of the gaussian kernel) looks like the following:

Code

Grab the code: http://web.media.mit.edu/~roys/src/CurveCSS.zip
(if the hosting dies - let me know)

Thanks for joining in
Roy.