CrowdManager.java

Go to the documentation of this file.

package crowdUser;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import myDataBases.MySQLbase;
import myDataBases.MediaBase;


public class CrowdManager {

  00042   // Fields
  00044 
  private List<Integer> greater;
  private List<Integer> smaller;
  private List<String> axes;
  private Integer comparisonsNumber;
  private String cfKey ;
  private Integer originalJobId;
  private String resultsUrl;
  private MediaBase dbMedia ;
  private MySQLbase dbSQL;
  private Refiner refiner;

  
  00094   // Constructors
  00110   public CrowdManager (Integer firstJobId, String key, String resultsUrl, MySQLbase baseSQL,
                                          MediaBase baseDIR, List<String> fieldsHardData)
  {
        this.comparisonsNumber = 0;
        this.resultsUrl = resultsUrl;
        this.greater = new ArrayList<Integer>();
        this.smaller = new ArrayList<Integer>();
        this.axes = new ArrayList<String>();
        this.originalJobId = firstJobId;
        this.cfKey = key;
        this.dbSQL = baseSQL;
        this.dbMedia = baseDIR ;
        this.refiner = new Refiner(fieldsHardData);
  };
  
  00126   // Methods
  00128   
  public Integer createNewJob()
  {
        Integer id = 0;

        String postUrl = "http://api.crowdflower.c../..bs/" + this.originalJobId
                  + "/copy.json?key=" + this.cfKey ;
        // posting the request
        try
        {
              URL url = new URL(postUrl);
              HttpURLConnection conn = (HttpURLConnection) url.openConnection();
              // Building an HTTP request
              conn.setDoOutput(true);
              conn.setRequestMethod("POST");
              conn.connect();
              // getting the answer from CrowdFlower and reading the "id" field
              BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
              String line, idString = "";
              while ((line = rd.readLine()) != null)
                    if (line.contains('"'+"id"+'"'))
                          // we go to the index of ' "id": ' and read the number. In order no to take '"id":', 5 is added.
                          for (int i = line.lastIndexOf('"' +"id"+'"')+5; i<line.length(); i++)
                                if ( line.charAt(i) == ',' )
                                      break;
                                else
                                      idString += line.charAt(i) ;
              id = Integer.parseInt(idString);
              rd.close();
        } catch (Exception e) { e.printStackTrace(); }

        return id;
  }


  
  public void uploadHIT(Integer jobId)
  {
        this.comparisonsNumber += this.greater.size() ;
        String post = "";
        // writing the .csv file and the post String
        try{
              BufferedWriter fichier = new BufferedWriter
                              (new FileWriter("../data/HITdata/" + jobId + ".csv"));
              String line = "axis,idMedia1,idMedia2,urlMedia1,urlMedia2,miscData1,miscData2";
              fichier.write(line);
            fichier.newLine();
            post += line ;
              for (int i=0; i<this.greater.size(); i++)
              {
                    line = this.axes.get(i) + "," + this.greater.get(i) + "," + this.smaller.get(i) +
                        "," + this.dbMedia.getMedia(this.greater.get(i)) +
                        "," + this.dbMedia.getMedia(this.smaller.get(i)) +
                        ",\"" + this.dbMedia.getContent().get(this.greater.get(i)).get("miscData") +
                        "\",\"" + this.dbMedia.getContent().get(this.smaller.get(i)).get("miscData") + "\"";
                    fichier.write(line);
                  fichier.newLine();
                  post += "\n" + line ;
              }
            fichier.close();
            
        }
        catch (Exception e) { e.printStackTrace(); }
        
        // posting the file

        try
        {
              URL url = new URL("http://api.crowdflower.c../..bs/" + jobId +
                        "/upload.json?key=" + this.cfKey);
              HttpURLConnection conn = (HttpURLConnection) url.openConnection();
              // Building an HTTP request
              conn.setDoOutput(true);
              conn.setRequestMethod("PUT");
              conn.setRequestProperty("Content-Type", "text/csv");
              OutputStreamWriter osw = new OutputStreamWriter(conn.getOutputStream());
              osw.write(post);
              osw.flush();
              osw.close();
              
              // getting the answer from CrowdFlower
              BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
              String line;
              while ((line = rd.readLine()) != null) {
                  System.out.println(line);
              }
              
              rd.close();
        } catch (Exception e) { e.printStackTrace(); }

  }


  public String getHITresults(Integer jobId)
  {
        try
        {
              // creating connection
              String urlString = this.resultsUrl + "csv/" + jobId + ".txt";
              System.out.println(urlString);
              URL url = new URL(urlString);
              System.out.println("Opening connection to " + urlString + "...");
              // reading input
              InputStream is = url.openStream();
              System.out.flush();
              // creating results$JOB_ID.csv file
              FileOutputStream fos=null;
              fos = new FileOutputStream("../data/HITresults/results" + jobId + ".csv");
              // writing to file
              int oneChar, count=0;
              while ((oneChar=is.read()) != -1)
              {
                    fos.write(oneChar);
                    count++;
              }
              // close everything
              is.close();
              fos.close();
              System.out.println("csv results file downloaded, " + count + " byte(s) copied");
        }
      catch (Exception e) { e.printStackTrace(); }
      // return the name of the file minus ".csv"
        return "results" + jobId ;
  }
  
  
  public void readHITresults( String path )
  {
        List<String[]> entries ;
        // retrieve hard data from the csv file
        this.refiner.csv2List(path) ;
        entries = this.refiner.getData() ;
        this.dbSQL.insertHardDataInSQL(entries);
        // retrieve good data from the refiner
        this.refiner.getanotherlabel() ;
        entries = this.refiner.getData() ;
        this.dbSQL.insertResultsInSQL(entries,this.dbMedia) ;
  }


  // Yes, little friend / This is the end / My only friend / the end / Of everything that stands / the end
  // I'll never look into your eyes / again...
  public static void thisIsTheEnd(String antechamberUrl, String results)
  {
        try
        {
              URL url = new URL(antechamberUrl);
              HttpURLConnection conn = (HttpURLConnection) url.openConnection();
              // Building an HTTP request
              conn.setDoOutput(true);
              conn.setRequestMethod("POST");
              OutputStreamWriter osw = new OutputStreamWriter(conn.getOutputStream());
              String data = URLEncoder.encode("signal", "UTF-8") + "=" + URLEncoder.encode("sort_finished","UTF-8")
                  + "&" + URLEncoder.encode("payload", "UTF-8") + "=" + URLEncoder.encode(results, "UTF-8");
              osw.write(data);
              osw.flush();
              osw.close();
              // getting the answer from CrowdFlower
              BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
              String line;
              while ((line = rd.readLine()) != null) {
                  System.out.println(line);
              }
              rd.close();
        } catch (Exception e) { e.printStackTrace(); }
  }
  
  00359   // Accessor methods
  00361 
  public List<Integer> getGreater()
  { return this.greater ; }

  
  public List<Integer> getSmaller()
  { return this.smaller; }

  
  public List<String> getAxes()
  { return this.axes; }
  
  
  public Integer getComparisonsNumber()
  { return this.comparisonsNumber; }


}