diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..61f855f6 Binary files /dev/null and b/.DS_Store differ diff --git a/.idea/misc.xml b/.idea/misc.xml index b0c7b20c..e0d5b93f 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -29,7 +29,7 @@ - + diff --git a/.idea/modules.xml b/.idea/modules.xml index 3f546a40..46661171 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -6,4 +6,4 @@ - \ No newline at end of file + diff --git a/app/.DS_Store b/app/.DS_Store new file mode 100644 index 00000000..b6e51842 Binary files /dev/null and b/app/.DS_Store differ diff --git a/app/build.gradle b/app/build.gradle index 6df9a4c5..25b0f9c7 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -16,6 +16,12 @@ android { proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' } } + + //Start - For android testing + testOptions { + unitTests.returnDefaultValues = true + } + //End - For android testing } dependencies { @@ -25,7 +31,23 @@ dependencies { testImplementation 'junit:junit:4.12' androidTestImplementation 'com.android.support.test:runner:1.0.2' androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' + + //for json testing + testImplementation 'org.json:json:20140107' + implementation 'com.google.firebase:firebase-core:16.0.4' implementation 'com.google.firebase:firebase-ml-vision:18.0.1' + implementation 'com.android.support:design:28.0.0' + + implementation group: 'info.debatty', name: 'java-string-similarity', version: '1.1.0' + + //CameraKIt dependencies + implementation 'com.camerakit:camerakit:1.0.0-beta3.9' + implementation 'com.camerakit:jpegkit:0.1.0' + implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.2.61' + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:0.24.0' + + //FTP dependencies + implementation 'commons-net:commons-net:3.6' } -apply plugin: 'com.google.gms.google-services' \ No newline at end of file +apply plugin: 'com.google.gms.google-services' diff --git a/app/google-services.json b/app/google-services.json new file mode 100644 index 00000000..49f189d0 --- /dev/null +++ b/app/google-services.json @@ -0,0 +1,42 @@ +{ + "project_info": { + "project_number": "148989252526", + "firebase_url": "https://gruppo1ingsw1819.firebaseio.com", + "project_id": "gruppo1ingsw1819", + "storage_bucket": "gruppo1ingsw1819.appspot.com" + }, + "client": [ + { + "client_info": { + "mobilesdk_app_id": "1:148989252526:android:eeb1c8433d22ba11", + "android_client_info": { + "package_name": "unipd.se18.ocrcamera" + } + }, + "oauth_client": [ + { + "client_id": "148989252526-o2repli8pa7bs3gncg27th5a51hkfum2.apps.googleusercontent.com", + "client_type": 3 + } + ], + "api_key": [ + { + "current_key": "AIzaSyAatL4jOpv6-IIpBlBc9TDffLgpNiyFJjI" + } + ], + "services": { + "analytics_service": { + "status": 1 + }, + "appinvite_service": { + "status": 1, + "other_platform_oauth_client": [] + }, + "ads_service": { + "status": 2 + } + } + } + ], + "configuration_version": "1" +} \ No newline at end of file diff --git a/app/src/.DS_Store b/app/src/.DS_Store new file mode 100644 index 00000000..3463049a Binary files /dev/null and b/app/src/.DS_Store differ diff --git a/app/src/main/.DS_Store b/app/src/main/.DS_Store new file mode 100644 index 00000000..0ba1f723 Binary files /dev/null and b/app/src/main/.DS_Store differ diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index b699e039..7419fafa 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -2,9 +2,13 @@ - + + + + + + - - + + + + + + + + + + + + - - - + + + + \ No newline at end of file diff --git a/app/src/main/java/.DS_Store b/app/src/main/java/.DS_Store new file mode 100644 index 00000000..85aaa335 Binary files /dev/null and b/app/src/main/java/.DS_Store differ diff --git a/app/src/main/java/unipd/.DS_Store b/app/src/main/java/unipd/.DS_Store new file mode 100644 index 00000000..f1d5021b Binary files /dev/null and b/app/src/main/java/unipd/.DS_Store differ diff --git a/app/src/main/java/unipd/se18/.DS_Store b/app/src/main/java/unipd/se18/.DS_Store new file mode 100644 index 00000000..d5dadbce Binary files /dev/null and b/app/src/main/java/unipd/se18/.DS_Store differ diff --git a/app/src/main/java/unipd/se18/ocrcamera/CameraActivity.java b/app/src/main/java/unipd/se18/ocrcamera/CameraActivity.java index c365a5f7..a4bdb8d0 100644 --- a/app/src/main/java/unipd/se18/ocrcamera/CameraActivity.java +++ b/app/src/main/java/unipd/se18/ocrcamera/CameraActivity.java @@ -1,614 +1,203 @@ package unipd.se18.ocrcamera; -import android.Manifest; -import android.app.Activity; import android.content.Context; import android.content.Intent; import android.content.SharedPreferences; -import android.content.pm.PackageManager; import android.graphics.Bitmap; import android.graphics.BitmapFactory; -import android.graphics.ImageFormat; -import android.graphics.SurfaceTexture; -import android.hardware.camera2.CameraAccessException; -import android.hardware.camera2.CameraCaptureSession; -import android.hardware.camera2.CameraCharacteristics; -import android.hardware.camera2.CameraDevice; -import android.hardware.camera2.CameraManager; -import android.hardware.camera2.CameraMetadata; -import android.hardware.camera2.CaptureRequest; -import android.hardware.camera2.TotalCaptureResult; -import android.hardware.camera2.params.StreamConfigurationMap; -import android.media.Image; -import android.media.ImageReader; -import android.os.Handler; -import android.os.HandlerThread; -import android.support.annotation.NonNull; -import android.support.v4.app.ActivityCompat; -import android.support.v7.app.AppCompatActivity; +import android.graphics.Matrix; +import android.hardware.Sensor; +import android.hardware.SensorEvent; +import android.hardware.SensorEventListener; +import android.hardware.SensorManager; import android.os.Bundle; -import android.util.Base64; +import android.support.design.widget.FloatingActionButton; +import android.support.v7.app.AppCompatActivity; import android.util.Log; -import android.util.Size; -import android.util.SparseIntArray; -import android.view.Surface; -import android.view.TextureView; import android.view.View; -import android.widget.Button; -import android.widget.Toast; - +import com.camerakit.CameraKitView; import java.io.File; import java.io.FileOutputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; /** * The Activity useful for making photos */ public class CameraActivity extends AppCompatActivity { - // Final shared variables - /** - * TAG used for logs - */ - private static final String TAG = "CameraActivity"; + private CameraKitView cameraKitView; + private static String orientationResult="P"; - /** - * Code for the permission requested - */ - private final int MY_CAMERA_REQUEST_CODE = 200; - - /** - * SparseIntArray used for the conversion from screen rotation to Bitmap orientation - */ - private static final SparseIntArray ORIENTATIONS = new SparseIntArray(); - static{ - ORIENTATIONS.append(Surface.ROTATION_0,90); - ORIENTATIONS.append(Surface.ROTATION_90,0); - ORIENTATIONS.append(Surface.ROTATION_180,270); - ORIENTATIONS.append(Surface.ROTATION_270,180); + public String getDir() { + return getExternalFilesDir(null).getAbsolutePath(); } - - // Shared variables - /** - * TextureView used for the camera preview - */ - private TextureView mCameraTextureView; - - /** - * The CameraDevice used to interact with the physical camera. - */ - private CameraDevice mCameraDevice; - - /** - * CaptureRequest.Builder used for the camera preview. - */ - private CaptureRequest.Builder mCaptureRequestBuilder; - - /** - * CameraCaptureSession used for the camera preview; - */ - private CameraCaptureSession mCameraCaptureSession; - - /** - * Size used for the camera preview - */ - private Size mPreviewSize; - /** - * ImageReader used for capturing images. - */ - private ImageReader mImageReader; - - /** - * Thread used for running tasks in background - */ - private HandlerThread mBackgroundHandlerThread; - - /** - * Handler associated to the background Thread - */ - private Handler mBackgroundHandler; - - - /** - * Semaphore to close the camera before exiting from the app + * onCreate method of the Android Activity Lifecycle + * @param savedInstanceState The Bundle of the last instance state saved + * @author Romanello Stefano */ - private Semaphore mCameraOpenCloseLock = new Semaphore(1); + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_camera); - /** - * Callback of the camera states - * author Pietro Prandini - */ - private CameraDevice.StateCallback mStateCallback = new CameraDevice.StateCallback() { - @Override - public void onOpened(@NonNull CameraDevice camera) { - Log.v(TAG, "CameraDevice.StateCallback -> onOpened"); - mCameraOpenCloseLock.release(); - mCameraDevice = camera; - createCameraPreview(); - } + cameraKitView = findViewById(R.id.cameraKitView); - @Override - public void onDisconnected(@NonNull CameraDevice camera) { - Log.v(TAG, "CameraDevice.StateCallback -> onDisconnected"); - mCameraOpenCloseLock.release(); - mCameraDevice.close(); - } + //Load sensor for understand the orientation of the phone + SensorManager sensorManager = (SensorManager) getSystemService(Context.SENSOR_SERVICE); + sensorManager.registerListener(new SensorEventListener() { + public int mOrientationDeg; //last rotation in degrees + private static final int _DATA_X = 0; + private static final int _DATA_Y = 1; + private static final int _DATA_Z = 2; + private int ORIENTATION_UNKNOWN = -1; - @Override - public void onError(@NonNull CameraDevice camera, int error) { - Log.v(TAG, "CameraDevice.StateCallback -> onError"); - mCameraOpenCloseLock.release(); - mCameraDevice.close(); - mCameraDevice = null; - - // Put the activity in onDestroy if required - Activity activity = getParent(); - if (null != activity) { - // Notify by a toast - runOnUiThread(new Runnable() { - @Override - public void run() { - Toast.makeText(CameraActivity.this, R.string.state_callback_onError, - Toast.LENGTH_LONG).show(); + @Override + public void onSensorChanged(SensorEvent event) + { + float[] values = event.values; + int orientation = ORIENTATION_UNKNOWN; + float X = -values[_DATA_X]; + float Y = -values[_DATA_Y]; + float Z = -values[_DATA_Z]; + float magnitude = X*X + Y*Y; + // Don't trust the angle if the magnitude is small compared to the y value + if (magnitude * 4 >= Z*Z) { + float OneEightyOverPi = 57.29577957855f; + float angle = (float)Math.atan2(-Y, X) * OneEightyOverPi; + orientation = 90 - (int)Math.round(angle); + // normalize to 0 - 359 range + while (orientation >= 360) { + orientation -= 360; } - }); - // Destroy the activity - activity.finish(); - } - } - }; - - /** - * SurfaceTextureListener used for the preview - * author Leonardo Rossi - */ - private TextureView.SurfaceTextureListener mSurfaceTextureListener = - new TextureView.SurfaceTextureListener() { - @Override - public void onSurfaceTextureAvailable(SurfaceTexture surface, int width, int height) { - Log.v(TAG, "mSurfaceTextureListener -> onSurfaceTextureAvailable"); - openCamera(); - } - - @Override - public void onSurfaceTextureSizeChanged(SurfaceTexture surface, int width, int height) { - - } + while (orientation < 0) { + orientation += 360; + } + } + //now we must figure out which orientation based on the degrees + if (orientation != mOrientationDeg) + { + mOrientationDeg = orientation; - @Override - public boolean onSurfaceTextureDestroyed(SurfaceTexture surface) { - return false; - } + View takePicButton = findViewById(R.id.take_photo_button); - @Override - public void onSurfaceTextureUpdated(SurfaceTexture surface) { + //figure out actual orientation + if(orientation == -1){//basically flat + } + else if(orientation <= 45 || orientation > 315){//round to 0 + Log.d("Sensor", "P"); //Portrait + orientationResult="P"; + takePicButton.setRotation(0); //rotate take picture button + } + else if(orientation > 45 && orientation <= 135){//round to 90 + Log.d("Sensor", "LR"); //LandscapeRight + orientationResult="LR"; + takePicButton.setRotation(270); + } + else if(orientation > 135 && orientation <= 225){//round to 180 + Log.d("Sensor", "PU"); //PortraitUpside + orientationResult="PU"; + takePicButton.setRotation(180); + } + else if(orientation > 225 && orientation <= 315){//round to 270 + Log.d("Sensor", "LL"); //LandscapeLeft + orientationResult="LL"; + takePicButton.setRotation(90); + } - } - }; + } + } - /** - * onCreate method of the Android Activity Lifecycle - * @param savedInstanceState The Bundle of the last instance state saved - * @modify mButtonTakePhoto The Button used for taking photo - * @modify mButtonLastPhoto The Button used for viewing the last photo captured - * @modify mCameraTextureView The TextureView used for the camera preview - * @author Leonardo Rossi - */ - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - setContentView(R.layout.activity_camera); + @Override + public void onAccuracyChanged(Sensor sensor, int accuracy) { + } + }, sensorManager.getDefaultSensor(Sensor.TYPE_ACCELEROMETER), SensorManager.SENSOR_DELAY_NORMAL); - // Initializing of the UI components - mCameraTextureView = findViewById(R.id.camera_view); - mCameraTextureView.setSurfaceTextureListener(mSurfaceTextureListener); - Button mButtonTakePhoto = findViewById(R.id.take_photo_button); + FloatingActionButton mButtonTakePhoto = findViewById(R.id.take_photo_button); mButtonTakePhoto.setOnClickListener(new View.OnClickListener() { @Override public void onClick(View v) { + SharedPreferences prefs = getSharedPreferences("prefs", MODE_PRIVATE); + SharedPreferences.Editor edit = prefs.edit(); + edit.putString("text", null); + edit.putString("imageDataPath", null); + edit.apply(); takePhoto(); } }); - Button mButtonLastPhoto = findViewById(R.id.last_photo_button); - mButtonLastPhoto.setOnClickListener(new View.OnClickListener() { - @Override - public void onClick(View v) { - lastPhoto(); - } - }); } /** - * onResume method of the Android Activity lifecycle - * @modify mCameraTextureView The TextureView used for the camera preview. - * @author Pietro Prandini + * Takes a photo, saves it inside internal storage and resets the last extracted text + * + * @modify SharedPreferences + * @author Romanello Stefano - modified by Leonardo Rossi */ - @Override - protected void onResume() { - super.onResume(); - Log.v(TAG,"onResume"); - startBackgroundThread(); - if (mCameraTextureView.isAvailable()) { - Log.v(TAG, "onResume -> cameraPreview is available"); - openCamera(); - } else { - Log.v(TAG, "onResume -> cemaraPreview is not available"); - mCameraTextureView.setSurfaceTextureListener(mSurfaceTextureListener); - } - } + private void takePhoto() { + cameraKitView.captureImage(new CameraKitView.ImageCallback() { + @Override + public void onImage(CameraKitView cameraKitView, final byte[] photo) { - /** - * onPause method of the Android Activity lifecycle - * @author Pietro Prandini - */ - @Override - protected void onPause() { - Log.v(TAG,"onPause"); - closeCamera(); - stopBackgroundThread(); - super.onPause(); - } + Bitmap bitmapImage = BitmapFactory.decodeByteArray(photo, 0, photo.length, null); - /** - * Opens a connection with a camera if it is permitted, otherwise return. - *

The method open a connection with the frontal camera if the permission is granted - * otherwise return

- * @modify Copy the Id of the frontal camera in variable cameraId - * @modify open the connection with the frontal camera - * @modify create a Log.e "connected" if the connection succeed - * @modify create a Log.e "impossible to open the camera" if the CameraAccessExeption succeed - * @author Giovanni Furlan - */ - private void openCamera() { - Log.v(TAG, "Open the Camera"); - - //Permission handling - if (ActivityCompat.checkSelfPermission(this, Manifest.permission.CAMERA) - != PackageManager.PERMISSION_GRANTED) { - String[] permissions = {Manifest.permission.CAMERA}; - ActivityCompat.requestPermissions(this, permissions, MY_CAMERA_REQUEST_CODE); - return; - } + //Image rotation + switch (orientationResult) + { + case "LR": bitmapImage=rotateImage(bitmapImage,90); break; + case "LL": bitmapImage=rotateImage(bitmapImage,270); break; + case "PU": bitmapImage=rotateImage(bitmapImage,180); break; + default: break; + } - //Camera opening process - CameraManager manager = (CameraManager) getSystemService(Context.CAMERA_SERVICE); - String cameraId; - try { - cameraId = manager.getCameraIdList()[0]; - CameraCharacteristics mCameraCharacteristics = manager.getCameraCharacteristics(cameraId); - StreamConfigurationMap stremCfgMap = mCameraCharacteristics.get( - CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); - if (stremCfgMap != null) { - mPreviewSize = stremCfgMap.getOutputSizes(SurfaceTexture.class)[0]; - } - if (!mCameraOpenCloseLock.tryAcquire(2500, TimeUnit.MILLISECONDS)) { - throw new RuntimeException("Time out waiting to lock camera opening."); - } - manager.openCamera(cameraId, mStateCallback, mBackgroundHandler); - } - catch(CameraAccessException e) { - Log.e(TAG, "impossible to open the camera"); - } catch (InterruptedException e) { - throw new RuntimeException("Interrupted while trying to lock camera opening.", e); - } + //Temporary stores the captured photo into a file that will be used from the Camera Result activity + String filePath= tempFileImage(CameraActivity.this, bitmapImage,"capturedImage"); - Log.e(TAG, "connected"); - } + SharedPreferences prefs = getSharedPreferences("prefs", MODE_PRIVATE); + SharedPreferences.Editor edit = prefs.edit(); + edit.putString("imagePath", filePath.trim()); + edit.apply(); - /** - * Controls the output of the permissions requests. - * @param requestCode The code assigned to the request - * @param permissions The list of the permissions requested - * @param grantResults The results of the requests - * @author Pietro Prandini - */ - @Override - public void onRequestPermissionsResult(int requestCode, @NonNull String permissions[], - @NonNull int[] grantResults) { - switch (requestCode) { - case MY_CAMERA_REQUEST_CODE: { - if(grantResults.length == 0 || grantResults[0] != PackageManager.PERMISSION_GRANTED) { - // Notify by a toast - runOnUiThread(new Runnable() { - @Override - public void run() { - Toast.makeText(CameraActivity.this, - R.string.permissions_not_granted, Toast.LENGTH_LONG).show(); - } - }); - // Destroy the activity - finish(); - } + //An intent that will launch the activity that will analyse the photo + Intent i = new Intent(CameraActivity.this, ResultActivity.class); + startActivity(i); } - } + }); } - /** - * Create the Handler useful for the camera preview - * @modify mBackgroundHandlerThread The HandlerThread useful for the camera background - * operations Handler - * @modify mBackgroundHandler The Handler useful for the camera background operations - * @author Pietro Prandini - */ - private void startBackgroundThread() - { - Log.v(TAG, "Start the background Handler Thread"); - mBackgroundHandlerThread = new HandlerThread("Camera preview"); - mBackgroundHandlerThread.start(); - mBackgroundHandler = new Handler(mBackgroundHandlerThread.getLooper()); - } - /** - * Stop the Handler useful for the camera preview - * @modify mBackgroundHandlerThread The HandlerThread useful for the camera background - * operations Handler - * @modify mBackgroundHandler The Handler useful for the camera background operations - * @author Pietro Prandini - */ - private void stopBackgroundThread() - { - Log.v(TAG, "Stop the background Handler Thread"); - mBackgroundHandlerThread.quitSafely(); - try { - mBackgroundHandlerThread.join(); - mBackgroundHandlerThread = null; - mBackgroundHandler = null; - } catch (InterruptedException e) { - e.printStackTrace(); - } + @Override + protected void onStart() { + super.onStart(); + cameraKitView.onStart(); } - /** - * Creates the preview of the camera capture frames. - * @modify mCameraDevice The CameraDevice associated to the camera used - * @modify mCaptureRequestBuilder The CaptureRequest.Builder that build the request for the - * camera capturing images - * @modify mCameraCaptureSession The CameraCaptureSession that control the capture session - * of the camera - * @author Pietro Prandini - */ - private void createCameraPreview() { - final String mTAG = "createCameraPreview -> "; - final String mTAGCaptureSession = mTAG + "createCaptureSession -> "; - Log.v(TAG, "Start the creation of camera preview"); - - try - { - // Instance of the Surface tools useful for the camera preview - SurfaceTexture mSurfaceTexture = null; - if(mCameraTextureView.isAvailable()) - { - mSurfaceTexture = mCameraTextureView.getSurfaceTexture(); - } - - // Check if the SurfaceTexture is not null - if (mSurfaceTexture == null) - { - // There is a bug in the SurfaceTexture creation - Log.v(TAG, mTAG + "The SurfaceTexture is null: there is a bug in the" + - " SurfaceTexture creation."); - - // Notify by a toast - runOnUiThread(new Runnable() - { - @Override - public void run() - { - Toast.makeText(CameraActivity.this, R.string.surface_texture_bug, - Toast.LENGTH_LONG).show(); - } - }); - - // The app can't continue: launch onDestroy method - finish(); - } - - // Set the size of the default buffer same as the size of the camera preview - mSurfaceTexture.setDefaultBufferSize(mPreviewSize.getWidth(), mPreviewSize.getHeight()); - - - //Output Surface - Surface mSurface = new Surface(mSurfaceTexture); - - // Set up the mCaptureRequestBuilder with the output surface mSurface - mCaptureRequestBuilder = mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW); - mCaptureRequestBuilder.addTarget(mSurface); - - Log.v(TAG, mTAG + "Creating the camera preview"); - // Create a camera preview - mCameraDevice.createCaptureSession(Arrays.asList(mSurface), new CameraCaptureSession.StateCallback() { - @Override - public void onConfigured(@NonNull CameraCaptureSession session) { - Log.v(TAG, mTAGCaptureSession + "onConfigured"); - // Check: if the camera is closed return - if(mCameraDevice == null) { - return; - } - mCameraCaptureSession = session; - updatePreview(); - } - - @Override - public void onConfigureFailed(@NonNull CameraCaptureSession session) { - Log.v(TAG, mTAGCaptureSession + "onConfigureFailed"); - - // Notify by a toast - runOnUiThread(new Runnable() { - @Override - public void run() { - Toast.makeText(CameraActivity.this, R.string.preview_failed, - Toast.LENGTH_LONG).show(); - } - }); - } - }, null); - } - catch (CameraAccessException e) - { - e.printStackTrace(); - } - + @Override + protected void onResume() { + super.onResume(); + cameraKitView.onResume(); } - /** - * Updates the preview of the camera frames. - * @modify mCameraDevice The CameraDevice associated to the camera used - * @modify mCaptureRequestBuilder The CaptureRequest.Builder that build the request - * for the camera capturing images - * @modify mCameraCaptureSession The CameraCaptureSession that control the capture session - * of the camera - * @author Pietro Prandini - */ - private void updatePreview() { - CaptureRequest mCaptureRequest; - // Check for every external call - if(mCameraDevice == null) - { - runOnUiThread(new Runnable() { - @Override - public void run() { - Toast.makeText(CameraActivity.this,R.string.update_preview_failed, - Toast.LENGTH_LONG).show(); - } - }); - return; - } - - // Auto focus should be continuous for camera preview. - mCaptureRequestBuilder.set(CaptureRequest.CONTROL_MODE, - CaptureRequest.CONTROL_MODE_AUTO); - try - { - // Start the camera preview - mCaptureRequest = mCaptureRequestBuilder.build(); - mCameraCaptureSession.setRepeatingRequest(mCaptureRequest, null, mBackgroundHandler); - } - catch (CameraAccessException e) - { - e.printStackTrace(); - } - + @Override + protected void onPause() { + cameraKitView.onPause(); + super.onPause(); } - /** - * Takes a photo. - *

Saves the captured photo, previously converted into Base64 String, into the current - * activity sharedPreferences

- * @modify mCameraDevice - * @author Alberto Valente, Taulant Bullaku - */ - private void takePhoto() { - - if(mCameraDevice == null) - { - Log.e("cameraDevice", "cameraDevice is null"); - return; - } - - CameraManager manager = (CameraManager) getSystemService(Context.CAMERA_SERVICE); - - try - { - CameraCharacteristics characteristics = manager.getCameraCharacteristics(mCameraDevice.getId()); - Size[] jpegSizes = Objects.requireNonNull(characteristics.get( - CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)).getOutputSizes(ImageFormat.JPEG); - int width = 640; - int height = 480; - if(jpegSizes != null && jpegSizes.length > 0) - { - width = jpegSizes[0].getWidth(); - height = jpegSizes[0].getHeight(); - } - - mImageReader = ImageReader.newInstance(width, height, ImageFormat.JPEG, 1); - - ImageReader.OnImageAvailableListener readerListener = new ImageReader.OnImageAvailableListener() - { - @Override - public void onImageAvailable(ImageReader reader) { - //acquires the last image and delivers it to a buffer - Image image = reader.acquireLatestImage(); - ByteBuffer buffer = image.getPlanes()[0].getBuffer(); - byte[] photoByteArray = new byte[buffer.capacity()]; - buffer.get(photoByteArray); - - //Converts the byte array related to the given photo to a String in Base64 format - String photoBitmapToString = Base64.encodeToString(photoByteArray, Base64.DEFAULT); - - //Create sharedPref file to save the Bitmap of last taken photo in a String form - SharedPreferences sharedPref = getPreferences(Context.MODE_PRIVATE); - SharedPreferences.Editor editor = sharedPref.edit(); - editor.putString("photoBitmap", photoBitmapToString); - editor.commit(); - - //@author Leonardo Rossi - //Temporary stores the caprured photo into a file that will be used from the Camera Result activity - Bitmap bmp = BitmapFactory.decodeByteArray(photoByteArray, 0, photoByteArray.length); - String filePath= tempFileImage(CameraActivity.this, bmp,"capturedImage"); - - //An intent that will launch the activity that will analyse the photo - Intent i = new Intent(CameraActivity.this, ResultActivity.class); - i.putExtra("imageDataPath", filePath); - startActivity(i); - } - }; - mImageReader.setOnImageAvailableListener(readerListener, mBackgroundHandler); - - //@author Leonardo Rosi - - //Output surfaces - List outputSurface = new ArrayList<>(2); - outputSurface.add(mImageReader.getSurface()); - outputSurface.add(new Surface(mCameraTextureView.getSurfaceTexture())); - - //Creation of a capture request to take a photo from the camera - final CaptureRequest.Builder captureBuilder = mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE); - captureBuilder.addTarget(mImageReader.getSurface()); - captureBuilder.set(CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_AUTO); - - //Check orientation base on device - int rotation = getWindowManager().getDefaultDisplay().getRotation(); - captureBuilder.set(CaptureRequest.JPEG_ORIENTATION,ORIENTATIONS.get(rotation)); - - final CameraCaptureSession.CaptureCallback captureListener = new CameraCaptureSession.CaptureCallback() { - @Override - public void onCaptureCompleted(@NonNull CameraCaptureSession session, - @NonNull CaptureRequest request, @NonNull TotalCaptureResult result) { - super.onCaptureCompleted(session, request, result); - createCameraPreview(); - } - }; - - mCameraDevice.createCaptureSession(outputSurface, new CameraCaptureSession.StateCallback() { - @Override - public void onConfigured(@NonNull CameraCaptureSession cameraCaptureSession) { - try { - cameraCaptureSession.capture(captureBuilder.build(),captureListener, mBackgroundHandler); - } catch (CameraAccessException e) { - e.printStackTrace(); - } - } + @Override + protected void onStop() { + cameraKitView.onStop(); + super.onStop(); + } - @Override - public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession) - { - } - },mBackgroundHandler); - } catch(CameraAccessException e) { - e.printStackTrace(); - } + @Override + public void onRequestPermissionsResult(int requestCode, String[] permissions, int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + cameraKitView.onRequestPermissionsResult(requestCode, permissions, grantResults); } + /** * Stores the captured image into a temporary file useful to pass large data between activities * and returns the file's path. @@ -618,7 +207,8 @@ public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession * @return The files path * @author Leonardo Rossi */ - private String tempFileImage(Context context, Bitmap bitmap, String name) { + private String tempFileImage(Context context, Bitmap bitmap, String name) + { File outputDir = context.getCacheDir(); File imageFile = new File(outputDir, name + ".jpg"); @@ -636,64 +226,25 @@ private String tempFileImage(Context context, Bitmap bitmap, String name) { return imageFile.getAbsolutePath(); } - /** - * Show the last photo and OCR text taken from preference - * @author Giovanni Furlan - */ - private void lastPhoto() { - SharedPreferences preferences = getPreferences(MODE_PRIVATE); - - String OCRText = preferences.getString("text", null); - String photoBitmapToString = preferences.getString("photoBitmap", null); - - if(photoBitmapToString != null) { - byte[] photoByteArray = Base64.decode(photoBitmapToString, Base64.DEFAULT); - Bitmap bmp = BitmapFactory.decodeByteArray(photoByteArray, 0, photoByteArray.length); - String filePath = tempFileImage(CameraActivity.this, bmp, "capturedImage"); - Intent i = new Intent(CameraActivity.this, ResultActivity.class); - i.putExtra("imageDataPath", filePath); - i.putExtra("text", OCRText); - startActivity(i); - } else { - Toast.makeText(CameraActivity.this, "No preview photo taken", Toast.LENGTH_SHORT).show(); - } - } /** - * Saves a photo previously taken. + * Rotate the bitmap image of the angle + * @param source the image + * @param angle angle of rotation + * @return Bitmap image rotated */ - private void savePhoto() { - + public static Bitmap rotateImage(Bitmap source, int angle) { + Matrix matrix = new Matrix(); + matrix.postRotate(angle); + return Bitmap.createBitmap(source, 0, 0, source.getWidth(), source.getHeight(), + matrix, true); } - /** - * Closes resources related to the camera. - * @modify mCameraDevice The CameraDevice associated to the camera used - * @modify mCameraCaptureSession The CameraCaptureSession that control the capture session of the camera - * @modify mImageReader The ImageReader useful for storing the photo captured - * @author Pietro Prandini - */ - private void closeCamera() { - try { - mCameraOpenCloseLock.acquire(); - if (null != mCameraCaptureSession) { - mCameraCaptureSession.close(); - mCameraCaptureSession = null; - } - if (null != mCameraDevice) { - mCameraDevice.close(); - mCameraDevice = null; - } - if (null != mImageReader) { - mImageReader.close(); - mImageReader = null; - } - } catch (InterruptedException e) { - throw new RuntimeException("Interrupted while trying to lock camera closing.", e); - } - finally { - mCameraOpenCloseLock.release(); - } - } + + + + } + + diff --git a/app/src/main/java/unipd/se18/ocrcamera/DeviceOrientation.java b/app/src/main/java/unipd/se18/ocrcamera/DeviceOrientation.java new file mode 100644 index 00000000..aa3abe95 --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/DeviceOrientation.java @@ -0,0 +1,108 @@ +package unipd.se18.ocrcamera; + +import android.hardware.Sensor; +import android.hardware.SensorEvent; +import android.hardware.SensorEventListener; +import android.hardware.SensorManager; +import android.support.media.ExifInterface; + +/** + * This class use the gyroscope to get the camera orientation + * @author Giovanni Furlan (g2) + * + */ +public class DeviceOrientation { + private final int ORIENTATION_PORTRAIT = ExifInterface.ORIENTATION_ROTATE_90; // 6 + private final int ORIENTATION_LANDSCAPE_REVERSE = ExifInterface.ORIENTATION_ROTATE_180; // 3 + private final int ORIENTATION_LANDSCAPE = ExifInterface.ORIENTATION_NORMAL; // 1 + private final int ORIENTATION_PORTRAIT_REVERSE = ExifInterface.ORIENTATION_ROTATE_270; // 8 + + private int smoothness = 1; + private float averagePitch = 0; + private float averageRoll = 0; + private int orientation = ORIENTATION_PORTRAIT; + + private float[] pitches; + private float[] rolls; + + DeviceOrientation() { + pitches = new float[smoothness]; + rolls = new float[smoothness]; + } + + SensorEventListener getEventListener() { + return sensorEventListener; + } + + public int getOrientation() { + return orientation; + } + + private SensorEventListener sensorEventListener = new SensorEventListener() { + float[] mGravity; + float[] mGeomagnetic; + + @Override + public void onSensorChanged(SensorEvent event) { + if (event.sensor.getType() == Sensor.TYPE_ACCELEROMETER) + mGravity = event.values; + if (event.sensor.getType() == Sensor.TYPE_MAGNETIC_FIELD) + mGeomagnetic = event.values; + if (mGravity != null && mGeomagnetic != null) { + float R[] = new float[9]; + float I[] = new float[9]; + boolean success = SensorManager.getRotationMatrix(R, I, mGravity, mGeomagnetic); + if (success) { + float orientationData[] = new float[3]; + SensorManager.getOrientation(R, orientationData); + averagePitch = addValue(orientationData[1], pitches); + averageRoll = addValue(orientationData[2], rolls); + orientation = calculateOrientation(); + } + } + } + + @Override + public void onAccuracyChanged(Sensor sensor, int accuracy) { + // TODO Auto-generated method stub + + } + }; + + private float addValue(float value, float[] values) { + value = (float) Math.round((Math.toDegrees(value))); + float average = 0; + for (int i = 1; i < smoothness; i++) { + values[i - 1] = values[i]; + average += values[i]; + } + values[smoothness - 1] = value; + average = (average + value) / smoothness; + return average; + } + + private int calculateOrientation() { + // finding local orientation dip + if (((orientation == ORIENTATION_PORTRAIT || orientation == ORIENTATION_PORTRAIT_REVERSE) + && (averageRoll > -30 && averageRoll < 30))) { + if (averagePitch > 0) + return ORIENTATION_PORTRAIT_REVERSE; + else + return ORIENTATION_PORTRAIT; + } else { + // divides between all orientations + if (Math.abs(averagePitch) >= 30) { + if (averagePitch > 0) + return ORIENTATION_PORTRAIT_REVERSE; + else + return ORIENTATION_PORTRAIT; + } else { + if (averageRoll > 0) { + return ORIENTATION_LANDSCAPE_REVERSE; + } else { + return ORIENTATION_LANDSCAPE; + } + } + } + } +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/DownloadDbActivity.java b/app/src/main/java/unipd/se18/ocrcamera/DownloadDbActivity.java new file mode 100644 index 00000000..329aec91 --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/DownloadDbActivity.java @@ -0,0 +1,35 @@ +package unipd.se18.ocrcamera; + + +import android.support.v7.app.AppCompatActivity; +import android.os.Bundle; +import android.view.View; +import android.widget.Button; + + + +public class DownloadDbActivity extends AppCompatActivity { + + private Button clickButton; + + /** + * Instantiate the UI elements + * + * @author Stefano Romanello (g3) + */ + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_download_db); + + ///Load UI element + clickButton = (Button) findViewById(R.id.downloadDbButton); + clickButton.setOnClickListener( new View.OnClickListener() { + @Override + public void onClick(View v) { + PhotoDownloadTask task = new PhotoDownloadTask(DownloadDbActivity.this); + task.execute(); + } + }); + } +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/ManualTestOnSinglePhoto.java b/app/src/main/java/unipd/se18/ocrcamera/ManualTestOnSinglePhoto.java new file mode 100644 index 00000000..59c468ad --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/ManualTestOnSinglePhoto.java @@ -0,0 +1,185 @@ +package unipd.se18.ocrcamera; + +import android.content.Intent; +import android.content.SharedPreferences; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Matrix; +import android.os.Bundle; +import android.support.annotation.Nullable; +import android.support.v7.app.AppCompatActivity; +import android.util.Log; +import android.view.View; +import android.widget.Button; +import android.widget.EditText; +import android.widget.TextView; + +import info.debatty.java.stringsimilarity.Damerau; +import unipd.se18.ocrcamera.recognizer.OCR; +import unipd.se18.ocrcamera.recognizer.OCRListener; +import unipd.se18.ocrcamera.recognizer.TextRecognizer; + +import static unipd.se18.ocrcamera.recognizer.TextRecognizer.getTextRecognizer; + +/** + * Class that compare the image stored image in preferences with the same image rotate + * of a certain angle given by the user, showing the similarity + */ + +public class ManualTestOnSinglePhoto extends AppCompatActivity { + + /** + * Text get from OCR of the last image taken + */ + private String startingOCRText; + /** + * Modification variable + */ + private int angleRotation; + + /** + * UI + */ + private TextView degreeTextView; + private TextView confidenceTextView; + private TextView differenceLengthTextView; + private TextView foundTextView; + + /** + * Listener used by the extraction process to notify results + */ + private OCRListener ocrListener = new OCRListener() { + @Override + public void onTextRecognized(String text) { + /* + Text correctly recognized + Initialize Ui + Start compare OCRText of the modified photo with the one with no rotation + */ + setContentView(R.layout.activity_manual_test_result); + degreeTextView = findViewById(R.id.degreeTextView); + confidenceTextView =findViewById(R.id.confidenceTextView); + differenceLengthTextView = findViewById(R.id.differenceLengthTextView); + foundTextView = findViewById(R.id.foundTextView); + + setResult(text); + } + + @Override + public void onTextRecognizedError(int code) { + /* + Text not correctly recognized + Start CameraActivity to take a new photo + */ + Intent cameraActivity = new Intent(ManualTestOnSinglePhoto.this, CameraActivity.class); + startActivity(cameraActivity); + } + }; + + + @Override + public void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + //UI initialization + setContentView(R.layout.activity_manual_test); + final EditText degreeEditText = findViewById(R.id.editText1); + + + //Get image path and text of the last image from preferences + SharedPreferences lastImagePref = this.getSharedPreferences("prefs", MODE_PRIVATE); + String pathImage = lastImagePref.getString("imagePath", null); + startingOCRText = lastImagePref.getString("text", null); + + final Bitmap photo = BitmapFactory.decodeFile(pathImage); + + Button confirmButton = findViewById(R.id.btnConfirm); + confirmButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + + try { + angleRotation = Integer.parseInt(degreeEditText.getText().toString()); + + if (photo == null) + throw new PhotoNullException(); + + //Photo changes + Bitmap photoToCompare = rotateImage(photo, angleRotation); + + // Instance of an OCR recognizer + OCR ocrProcess = getTextRecognizer(TextRecognizer.Recognizer.mlKit, + ocrListener); + + // Runs the operations of text extraction + ocrProcess.getTextFromImg(photoToCompare); + + + } catch (PhotoNullException e) { + Log.e(null, getResources().getString(R.string.errorLog1)); + Intent takeANewPhoto = new Intent(ManualTestOnSinglePhoto.this, CameraActivity.class); + startActivity(takeANewPhoto); + } + } + }); + } + + /**Based on the text received and startingOCRTText, analise information like confidence + * and show them on textView + * @param text string got from OCR + * @modify degreeTextView, confidenceTextView, differenceLengthTextView and foundTextView + */ + private void setResult(String text){ + String differenceLength = Integer.toString(startingOCRText.length()-text.length()); + String confidence = Double.toString(compareStrings(startingOCRText,text)); + + degreeTextView.setText(angleRotation); + confidenceTextView.setText(confidence); + foundTextView.setText(text); + differenceLengthTextView.setText(differenceLength); + } + + /** + * Rotate the bitmap image of the angle + * @param source the image + * @param angle angle of rotation + * @return Bitmap image rotated + */ + public static Bitmap rotateImage (Bitmap source,int angle) + { + Matrix matrix = new Matrix(); + matrix.postRotate(angle); + return Bitmap.createBitmap(source, 0, 0, source.getWidth(), source.getHeight(), + matrix, true); + } + + + /** Calculate Damerau-Levenshtein distance between the strings and return similarity + * @param firstString string to split and search in secondString + * @param secondString string where to search + * @return percentage of similarity + * see the link for more information {@link link} + */ + public static double compareStrings (String firstString, String secondString) { + + Damerau damerau = new Damerau(); + //Percentage of distance based on firstString length + double distance = damerau.distance(firstString,secondString)/firstString.length()*100; + return Math.round(100-distance); + } +} + +/** + * New Exception + */ +class PhotoNullException extends Exception +{ + // Parameterless Constructor + PhotoNullException() {} + + // Constructor that accepts a message + PhotoNullException(String message) + { + super(message); + } +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/NavigatorActivity.java b/app/src/main/java/unipd/se18/ocrcamera/NavigatorActivity.java new file mode 100644 index 00000000..4388f673 --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/NavigatorActivity.java @@ -0,0 +1,66 @@ +package unipd.se18.ocrcamera; + +import android.content.Context; +import android.content.Intent; +import android.content.SharedPreferences; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.preference.PreferenceManager; +import android.support.v7.app.AppCompatActivity; +import android.os.Bundle; +import android.util.Log; + +import java.io.File; + +/* + This activity is the starting activity that chooses which activity to start + Author: Francesco Pham + */ +public class NavigatorActivity extends AppCompatActivity { + + /** + * TAG used for logs + */ + private static final String TAG = "NavigatorActivity"; + + @Override + protected void onCreate(Bundle savedInstanceState) { + + super.onCreate(savedInstanceState); + + //Get image path of last image + SharedPreferences prefs = getSharedPreferences("prefs", MODE_PRIVATE); + String pathImage = prefs.getString("imagePath", null); + + + Intent intent; + + /** + If already exists a photo, launch result activity to show it + with text attached - Author Luca Moroldo modified by Francesco Pham + **/ + if(pathImage != null) { + //load last extracted text + prefs = getSharedPreferences("prefs", MODE_PRIVATE); + String OCRText = prefs.getString("text", null); + + if(OCRText != null && !(OCRText.equals(""))) { + //An intent that will launch the activity + intent = new Intent(NavigatorActivity.this, ResultActivity.class); + + } + else { + Log.e(TAG, "Error retrieving last extracted text"); + intent = new Intent(NavigatorActivity.this, CameraActivity.class); + } + + } + else { + intent = new Intent(NavigatorActivity.this, CameraActivity.class); + } + + + startActivity(intent); + finish(); + } +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/OCRWrapper.java b/app/src/main/java/unipd/se18/ocrcamera/OCRWrapper.java deleted file mode 100644 index c442a730..00000000 --- a/app/src/main/java/unipd/se18/ocrcamera/OCRWrapper.java +++ /dev/null @@ -1,12 +0,0 @@ -package unipd.se18.ocrcamera; - -import android.graphics.Bitmap; - -interface OCRWrapper { - /** - * Wrapper for OCR libraries. Extract a text from a given image. - * @param img The image in a Bitmap format - * @return The String of the text recognized (empty String if nothing is recognized) - */ - String getTextFromImg(Bitmap img); -} diff --git a/app/src/main/java/unipd/se18/ocrcamera/PhotoDownloadTask.java b/app/src/main/java/unipd/se18/ocrcamera/PhotoDownloadTask.java new file mode 100644 index 00000000..88cfc022 --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/PhotoDownloadTask.java @@ -0,0 +1,253 @@ +package unipd.se18.ocrcamera; + +import android.app.Activity; +import android.content.Context; +import android.os.AsyncTask; +import android.os.Environment; +import android.util.Log; +import android.widget.Button; +import android.widget.ProgressBar; +import android.widget.ScrollView; +import android.widget.TextView; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPFile; +import org.apache.commons.net.ftp.FTPReply; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; + + +/** + * @author Leonardo Rossi (g2) and Stefano Romanello (g3) + */ +public class PhotoDownloadTask extends AsyncTask +{ + + private FTPClient ftp; + //private ArrayList messages; + private Context context; + private ProgressBar progressBar; + private TextView textViewProgress; + private Button button; + private Integer currentProgress; + private ScrollView scrollCurrentDownload; + private TextView txtViewCurrentDownload; + + protected static final String PHOTOS_FOLDER = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_PICTURES)+"/OCRCameraDB"; + private final String LOGINGINFORMATION_FILE = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS)+"/ingsoftwareftp.txt"; + private final String REMOTE_FOLDER = "/htdocs/foto/"; + private final String TAG = "FTP"; + DownloadDbActivity activity; + PhotoDownloadTask(Context context) { this.context = context; } + + @Override + protected void onPreExecute() + { + activity = (DownloadDbActivity) context; + ftp = new FTPClient(); + progressBar = activity.findViewById(R.id.progressBar); + textViewProgress = activity.findViewById(R.id.textViewProgress); + button = activity.findViewById(R.id.downloadDbButton); + scrollCurrentDownload = activity.findViewById(R.id.scrollView); + txtViewCurrentDownload = activity.findViewById(R.id.textViewCurrentDownload); + + //Reset download from the prevous downloads + disable button for preventing multiple downloads + currentProgress = 0; + button.setEnabled(false); + txtViewCurrentDownload.setText(""); + } + + @Override + protected Void doInBackground(Void... voids) + { + try + { + Boolean isConnected = connectToServer(); + + if (isConnected) + { + retrieveFiles(); + } + } + catch (Exception e) + { + sendMessageToUI(e.toString()); + } + + return null; + } + + @Override + protected void onPostExecute(Void params) + { + button.setEnabled(true); + } + + @Override + protected void onProgressUpdate(Integer... values) + { + //Update the progressBar + progressBar.setProgress(values[0]); + textViewProgress.setText("Status: " + values[0]+" of " + progressBar.getMax()); + } + + /** + * Tries to connect the FTPclient to the server + * @throws IOException if an error occurs during the connection to the server + * @return True if connected, false otherwise + */ + private Boolean connectToServer() throws IOException + { + //0 username + //1 password + //2 hostname + + String[] credentials = getFTPCredentials(); + + //Trying to connect to the server + ftp.connect(credentials[2]); + + //Logging in into the server + if (!ftp.login(credentials[0], credentials[1])) + { + ftp.logout(); + throw new IOException("Bad Credentials"); + } + + int reply = ftp.getReplyCode(); + //FTPReply stores a set of constants for FTP reply codes. + if (!FTPReply.isPositiveCompletion(reply)) + { + ftp.disconnect(); + throw new IOException("Bad Credentials"); + } + + //enter passive mode + ftp.enterLocalPassiveMode(); + //get system name + System.out.println("Remote system is " + ftp.getSystemType()); + //change current directory + ftp.changeWorkingDirectory(REMOTE_FOLDER); + System.out.println("Current directory is " + ftp.printWorkingDirectory()); + + return true; + } + + /** + * Retrieves files from the server + * @throws IOException if an error occurs while retrieving files from server + */ + private void retrieveFiles() throws IOException + { + //get list of filenames + FTPFile[] ftpFiles = ftp.listFiles(); + Log.d(TAG,"num file " + ftpFiles.length); + + if (ftpFiles != null && ftpFiles.length > 0) + { + progressBar.setMax(ftpFiles.length); + //Looping through files + for (FTPFile file: ftpFiles) + { + //Send current progress to "onProgressUpdate" + publishProgress(++currentProgress); + + if (!file.isFile()) + { + //Send message if object is not a file + sendMessageToUI("Skipped: not a file" +System.getProperty("line.separator")); + } + else + { + //Creation of the destination file for the image downloaded from the server + File destinationFile = new File(PHOTOS_FOLDER + "/" + file.getName()); + + //Don't download already downloaded files + if (!destinationFile.exists()) + { + OutputStream output; + output = new FileOutputStream(PHOTOS_FOLDER + "/" + file.getName()); + //get the file from the remote system + ftp.retrieveFile(file.getName(), output); + //close output stream + output.close(); + Log.d(TAG, "Downloaded: " + PHOTOS_FOLDER + "/" + file.getName()); + + //Send message if the file is not already downloaded + sendMessageToUI("Downloaded: " + file.getName() +System.getProperty("line.separator")); + } + else + { + //Send message if the file is already downloaded + sendMessageToUI("Skipped: " + file.getName() +System.getProperty("line.separator")); + } + } + } + } + + Log.d(TAG, "Finished"); + sendMessageToUI("Finished"); + ftp.logout(); + ftp.disconnect(); + } + + /** + * Update the scrollable textView in the UI + * @author Stefano Romanello + */ + private void sendMessageToUI(String message) + { + final String messageToSend = message; + activity.runOnUiThread(new Runnable() + { + @Override + public void run() + { + txtViewCurrentDownload.append(messageToSend); + scrollCurrentDownload.smoothScrollTo(0, txtViewCurrentDownload.getBottom()); + } + }); + } + /** + * Update the scrollable textView in the UI + * Obtain the credentials from the file + * @author Stefano Romanello + */ + private String[] getFTPCredentials() + { + FileInputStream is; + BufferedReader reader; + final File file = new File(LOGINGINFORMATION_FILE); + + ArrayList lines= new ArrayList(); + if (file.exists()) { + try { + is = new FileInputStream(file); + reader = new BufferedReader(new InputStreamReader(is)); + String line = reader.readLine(); + lines.add(line); + while(line != null){ + line = reader.readLine(); + lines.add(line); + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + Object[] objNames = lines.toArray(); + String[] outCredentials = Arrays.copyOf(objNames, objNames.length, String[].class); + return outCredentials; + } +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/PhotoTester.java b/app/src/main/java/unipd/se18/ocrcamera/PhotoTester.java new file mode 100644 index 00000000..bf63c939 --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/PhotoTester.java @@ -0,0 +1,763 @@ +package unipd.se18.ocrcamera; + +import android.graphics.Bitmap; +import android.util.Log; + +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import info.debatty.java.stringsimilarity.*; +import unipd.se18.ocrcamera.recognizer.OCR; +import unipd.se18.ocrcamera.recognizer.OCRListener; +import unipd.se18.ocrcamera.recognizer.TextRecognizer; +import info.debatty.java.stringsimilarity.CharacterSubstitutionInterface; +import info.debatty.java.stringsimilarity.WeightedLevenshtein; + +import static java.lang.Float.NaN; + +/** + * Class built to test the application's OCR comparing the goal text with the recognized text and + * providing a JSON report containing stats and results. + * @author Luca Moroldo (g3) - Francesco Pham (g3) + */ +public class PhotoTester { + + private static final String TAG = "PhotoTester"; + + /** + * Contains the available extensions for the test + */ + public static final String[] IMAGE_EXTENSIONS = {"jpeg", "jpg"}; + + /** + * Contains the base name of a photo used for the test + */ + public static final String PHOTO_BASE_NAME = "foto"; + + /** + * String used as file name for the report + */ + private static final String REPORT_FILENAME = "report.txt"; + + private ArrayList testElements = new ArrayList<>(); + + private TestListener testListener; + + //stores the path of the directory containing test files + private String dirPath; + + private String report; + + + + + + /** + * + * Load test elements (images + description) + * @param dirPath The path where the photos and descriptions are. + */ + public PhotoTester(String dirPath) { + File directory = getStorageDir(dirPath); + this.dirPath = directory.getPath(); + Log.v(TAG, "PhotoTester -> dirPath == " + dirPath); + + //create a TestElement object for each original photo - then link all the alterations to the relative original TestElement + for(File file : directory.listFiles()) { + + String filePath = file.getPath(); + String fileName = Utils.getFilePrefix(filePath); + + //if the file is not an alteration then create a test element for it + if(fileName.contains(PHOTO_BASE_NAME)) { + + String fileExtension = Utils.getFileExtension(filePath); + + //check if extension is available + if(Arrays.asList(IMAGE_EXTENSIONS).contains(fileExtension)) { + + //this file is an image -> get file path + String originalImagePath = file.getAbsolutePath(); + + //Each photo has a description.txt with the same filename - so when an image is found we know the description filename + String photoDesc= Utils.getTextFromFile(dirPath + "/" + fileName + ".txt"); + + //create test element giving filename, description and image path + //author Luca Moroldo - g3 + + JSONObject jsonPhotoDescription = null; + try { + jsonPhotoDescription = new JSONObject(photoDesc); + } catch(JSONException e) { + e.printStackTrace(); + Log.e(TAG, "PhotoTester constructor -> Error decoding JSON"); + } + if(jsonPhotoDescription != null) { + + TestElement originalTest = new TestElement(originalImagePath, jsonPhotoDescription, fileName); + + //associate the relative image path to each alteration of the original test if there is any + String[] alterationsFilenames = originalTest.getAlterationsNames(); + if(alterationsFilenames != null) { + for(String alterationFilename : alterationsFilenames) { + String alterationImagePath = dirPath + "/" + alterationFilename; + originalTest.setAlterationImagePath(alterationFilename, alterationImagePath); + } + } + + testElements.add(originalTest); + } + } + } + } + } + + + /** + * Get a File directory from a path String + * @param dirPath The path of the directory + * @return the file relative to the environment and the dirName + * @author Pietro Prandini (g2) + */ + private File getStorageDir(String dirPath) { + // Get the directory for the user's public pictures directory. + File file = new File(dirPath); + if(!file.isDirectory()) { + Log.e(TAG, file.getAbsolutePath() + "It's not a directory"); + } else { + Log.v(TAG, "Directory => " + file.getAbsolutePath()); + } + return file; + } + + + /** + * Elaborate tests using threads, stores the json report in string format to testReport.txt inside the directory given on construction + * @return String in JSON format with the test's report, each object is a single test named with the filename and contains: + * ingredients, tags, notes, original photo name, confidence and alterations (if any), each alteration contains alteration tags and alteration notes + * @author Luca Moroldo (g3) + */ + public String testAndReport() throws InterruptedException { + + + Log.i(TAG,"testAndReport started"); + long started = java.lang.System.currentTimeMillis(); + + final JSONObject fullJsonReport = new JSONObject(); + + //countDownLatch allows to sync this thread with the end of all the single tests + CountDownLatch countDownLatch = new CountDownLatch(testElements.size()); + + int max_concurrent_tasks = Runtime.getRuntime().availableProcessors(); + //leave a processor for the OS + if(max_concurrent_tasks > 1) { + max_concurrent_tasks--; + } + + Log.i(TAG, "max_concurrent_tasks == " + max_concurrent_tasks + " (number of the available cores)"); + + //Define a thread executor that will run a maximum of 'max_concurrent_tasks' simultaneously + ExecutorService executor = Executors.newFixedThreadPool(max_concurrent_tasks); + + for(TestElement singleTest : testElements) { + Runnable runnableTest = new RunnableTest(fullJsonReport, singleTest, countDownLatch); + executor.execute(runnableTest); + } + + //after shut down the executor will reject any new task + executor.shutdown(); + + //wait until all tests are completed - i.e. when CoundDownLatch.countDown() is called by each runnableTest + countDownLatch.await(); + + long ended = java.lang.System.currentTimeMillis(); + Log.i(TAG,"testAndReport ended (" + testElements.size() + " pics tested in " + (ended - started) + " ms)"); + + //insert tags stats to json report + String tagsStats = getTagsStatsString(); + try { + fullJsonReport.put("stats", tagsStats); + } catch(JSONException e) { + Log.e(TAG, "Failed to add tags stats to JSON report"); + } + + String fullReport = fullJsonReport.toString(); + + //write report to file + try { + writeReportToExternalStorage(fullReport, dirPath, REPORT_FILENAME); + } catch (IOException e) { + Log.e(TAG, "Error writing report to file."); + e.printStackTrace(); + } + + //save current report + this.report = fullReport; + + return fullReport; + } + + /** + * Save report to file + * + * @return true if report was correctly saved, false in case of error or if report is null + */ + public boolean saveReportToFile() { + + //check if report is not null + if(report == null) + return false; + + try { + writeReportToExternalStorage(report, dirPath, REPORT_FILENAME); + return true; + } catch (IOException e) { + Log.e(TAG, "Error writing report to file."); + e.printStackTrace(); + } + //error occurred + return false; + } + + public TestElement[] getTestElements() { + return testElements.toArray(new TestElement[0]); + } + public int getTestSize() { return testElements.size(); } + + /** + * Compare the list of ingredients extracted by OCR and the correct list of ingredients + * + * @param correct Correct list of ingredients loaded from file + * @param extracted List of ingredients extracted by the OCR + * @return Confidence percentage based on number of matched words, their similarity and order + * @author Francesco Pham credit to Stefano Romanello for Levenshtein library suggestion + */ + private float ingredientsTextComparison(String correct, String extracted){ + + extracted = extracted.toLowerCase(); //ignoring case + + //split words + String[] extractedWords = extracted.trim().split("[ ,-:./\\n\\r]+"); + String[] correctWords = correct.trim().split("[ ,-:./\\n\\r]+"); + + Log.i(TAG, "ingredientsTextComparison -> Start of comparing"); + Log.i(TAG, "ingredientsTextComparison -> correctWords.length == " + correctWords.length + ", extractedWords.length == " + extractedWords.length); + + float points = 0; //points are added each time a word is found + int maxPoints = 0; //maximum points which is the number of characters of all words with more than 3 characters + int posLastWordFound = 0; //index of the last word found + int consecutiveNotFound = 0; //consecutive words not found since last word found + final double similarityThreshold = 0.8; //threshold above which the word we are looking at is considered found + + WeightedLevenshtein levenshtein = new WeightedLevenshtein( + new CharacterSubstitutionInterface() { + public double cost(char c1, char c2) { + + // The cost for substituting 't' and 'r' is considered + // smaller as these 2 are located next to each other + // on a keyboard + if (c1 == 't' && c2 == 'r') { + return 0.5; + } + else if (c1 == 'q' && c2 == 'o') { + return 0.5; + } + else if (c1 == 'I' && c2 == 'l') { + return 0.5; + } + + // For most cases, the cost of substituting 2 characters + // is 1.0 + return 1.0; + } + }); + + //for each correct word + for (String word : correctWords) { + boolean found = false; + int index = posLastWordFound; + word = word.toLowerCase(); //ignoring case + + if (word.length() >= 3) { //ignoring non significant words with 1 or 2 characters + maxPoints += word.length(); + + for (int i = 0; i < extractedWords.length && !found; i++) { + //for each extracted words starting from posLastWordFound + index = (posLastWordFound + i) % extractedWords.length; + + //Calculate similarity and normalize + int maxLength = Math.max(word.length(),extractedWords[index].length()); + double similarity = 1.0 - levenshtein.distance(word,extractedWords[index])/maxLength; + + //if similarity grater than similarityThreshold the word is found + if (similarity > similarityThreshold) { + if (points == 0 || i < consecutiveNotFound + 10) { + points += word.length()*similarity; //assign points based on number of characters + } else { + //if word found is distant from posLastWordFound the ocr text isn't ordered, less points + points += (float) word.length()*similarity/2; + } + Log.d(TAG, "ingredientsTextComparison -> \"" + word + "\" == \"" + extractedWords[index] + "\" similarity="+similarity); + extractedWords[index] = ""; //remove found word + found = true; + } + } + } + + //taking into consideration words that are not properly separated (e.g. "cetarylalcohol") + if(!found && word.length() >= 6){ + for(int i=0; i \"" + word + "\" contained in \"" + extractedWords[index] + "\""); + extractedWords[index] = extractedWords[index].replace(word, ""); //remove found word + found = true; + } + } + } + + if(found){ + consecutiveNotFound = 0; + posLastWordFound = index; + } else { + consecutiveNotFound++; + } + } + float confidence = (points / maxPoints)*100; + Log.i(TAG, "ingredientsTextComparison -> confidence == " + confidence + " (%)"); + + //I found a test where the function returned NaN (the correct ingredient text was '-') - Luca Moroldo + if(confidence == NaN) { + confidence = 0; + } + return confidence; + + } + + /** + * Write the string given as argument to a file named with the fileName argument + * @param report text that will be saved to filename + * @param dirPath path to a directory with writing permissions + * @param fileName name of the file - will be overwritten if already exist + * @author Luca Moroldo (g3) + * @throws IOException + */ + private void writeReportToExternalStorage(String report, String dirPath, String fileName) throws IOException { + //Write report to report.txt + File file = new File(dirPath, fileName); + + file.createNewFile(); + FileOutputStream stream = null; + + try { + stream = new FileOutputStream(file); + + try { + stream.write(report.getBytes()); + } finally { + stream.close(); + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + Log.e(TAG, "File not found"); + } + } + + /** + * Class used to run a single test + * @author Luca Moroldo (g3) - Pietro Prandini (g2) + */ + public class RunnableTest implements Runnable { + /** + * Element of test to be analyzed + */ + private TestElement test; + + /** + * Report to be generated + */ + private JSONObject jsonReport; + + /** + * Synchronization construct for synchronizing with the other test threads + */ + private CountDownLatch countDownLatch; + + /** + * String where putting the correctIngredients provided by the database + */ + private String correctIngredients; + + /** + * Start runnable time variable + */ + private long started; + + /** + * Stop runnable time variable + */ + private long ended; + + /** + * Number of alterations to be processed + */ + private int alterations; + + /** + * Counter for the alterations analyzed + */ + private int alterationsAnalyzed; + + /** + * Object useful for synchronize critical statements + * (such as the increment of alterationsAnalyzed variable) + */ + private final Object lock = new Object(); + + /** + * Listener used by the OCR process + */ + OCRListener testOCRListener = new OCRListener() { + @Override + public void onTextRecognized(String text) { + // Process has got success -> processes the result + setTestResult(text); + } + + @Override + public void onTextRecognizedError(int code) { + // Process hasn't got success -> notifies to the result + String errorText = R.string.extraction_error + + " (" + R.string.error_code + code + ")"; + setTestResult(errorText); + } + }; + + /** + * @param jsonReport JSONObject containing tests data + * @param test element of a test - must contain an image path and ingredients fields + * @param countDownLatch used to signal the task completion + */ + RunnableTest(JSONObject jsonReport, TestElement test, CountDownLatch countDownLatch) { + this.jsonReport = jsonReport; + this.test = test; + this.countDownLatch = countDownLatch; + this.alterationsAnalyzed = 0; + } + + @Override + public void run() { + // Starts the runnable test + started = java.lang.System.currentTimeMillis(); + Log.d(TAG,"RunnableTest -> id \"" + Thread.currentThread().getId() + "\" started"); + + // Retrieves the ingredients provided by the database + correctIngredients = test.getIngredients(); + + // Instance of the OCR object + OCR ocrTestProcess = TextRecognizer.getTextRecognizer( + TextRecognizer.Recognizer.mlKit, + testOCRListener + ); + + // Retrieves the test pic + String imagePath = test.getImagePath(); + Bitmap testBitmap = Utils.loadBitmapFromFile(imagePath); + + // Launches the text extracting process + ocrTestProcess.getTextFromImg(testBitmap); + } + + /** + * Sets the Test result + * @param extractedIngredients The extracted ingredients from the OCR process + */ + private void setTestResult(String extractedIngredients) { + // Compares the text provided by the database and the text extracted + float confidence = ingredientsTextComparison(correctIngredients, extractedIngredients); + + //inserts results in the test element + test.setConfidence(confidence); + test.setRecognizedText(extractedIngredients); + + // evaluates alterations if any + String[] alterationsFileNames = test.getAlterationsNames(); + if(alterationsFileNames != null) { + // Sets the number of alterations + alterations = alterationsFileNames.length; + + // Processes the alterations + for(String alterationFilename : alterationsFileNames) { + analyzeAlteration(alterationFilename); + } + } else { + // No alterations -> TestElement analyzing completed + closingTest(); + } + } + + /** + * Analyzes an alteration + * @param alterationFilename The filename of the alteration to be analyzed + */ + private void analyzeAlteration(final String alterationFilename) { + // Gets the alterations pic + String alterationImagePath = test.getAlterationImagePath(alterationFilename); + Bitmap alterationBitmap = Utils.loadBitmapFromFile(alterationImagePath); + + // Listener useful to notify the alteration processing completed + final TestListener alterationResultListener = new TestListener() { + @Override + public void onTestFinished() { + // not useful in this case + } + + @Override + public void onAlterationAnalyzed() { + synchronized (lock) { + // +1 alteration analyzed + alterationsAnalyzed += 1; + + // Check if all the alterations is analyzed + if(alterations == alterationsAnalyzed) { + // All the alterations is analyzed -> closes the test + closingTest(); + } + } + } + }; + + // Listener useful for analyzing the alteration + final OCRListener alterationListener = new OCRListener() { + @Override + public void onTextRecognized(String text) { + // Alteration correctly analyzed + setAlterationsResult(alterationFilename,text,alterationResultListener); + } + + @Override + public void onTextRecognizedError(int code) { + // Error during the alteration analyzing + String errorText = R.string.extraction_error + + " (" + R.string.error_code + code + ")"; + setAlterationsResult(alterationFilename,errorText,alterationResultListener); + } + }; + + // Analyze the alteration pic if it's correctly retrieved + if(alterationBitmap != null) { + OCR ocrAlterationsProcess = TextRecognizer.getTextRecognizer( + TextRecognizer.Recognizer.mlKit, + alterationListener + ); + ocrAlterationsProcess.getTextFromImg(alterationBitmap); + } + } + + /** + * Sets the alterations result + * @param alterationFilename Filename of the alteration analyzed + * @param alterationExtractedIngredients Extracted ingredients + * @param alterationResultListener Listener used to synchronize the analyzing process + */ + private void setAlterationsResult(String alterationFilename, + String alterationExtractedIngredients, + TestListener alterationResultListener) { + // Compares the text provided by the database and the text extracted + float alterationConfidence = ingredientsTextComparison( + correctIngredients, + alterationExtractedIngredients + ); + + // Inserts evaluation + test.setAlterationConfidence(alterationFilename, alterationConfidence); + test.setAlterationRecognizedText(alterationFilename, alterationExtractedIngredients); + alterationResultListener.onAlterationAnalyzed(); + } + + /** + * Closes the analyzing process + */ + private void closingTest() { + // Adds the test element to the report + try { + addTestElement(jsonReport, test); + } catch (JSONException e) { + Log.e(TAG, "Failed to add test element '" + test.getFileName() + + " to json report"); + } + + //signals the end of this single test + countDownLatch.countDown(); + + //if the listener has been set then call onTestFinished function + if(testListener != null) { + synchronized (lock) { + testListener.onTestFinished(); + } + + } + // Closing log + ended = java.lang.System.currentTimeMillis(); + Log.d(TAG,"RunnableTest -> id \"" + Thread.currentThread().getId() + + "\" ended (runned for " + (ended - started) + " ms)"); + + } + } + + /** + * Set a listener whose function will be called at the end of each test + * @param testListener + */ + public void setTestListener(TestListener testListener) { + this.testListener = testListener; + } + + /** + * Add a single TestElement associated JSONReport inside the JSONObject jsonReport, + * multi-thread safe + * @param jsonReport the report containing tests in JSON format + * @param test element of a test + * @modify jsonReport + * @throws JSONException + * @author Luca Moroldo (g3) + */ + synchronized void addTestElement(JSONObject jsonReport, TestElement test) throws JSONException { + jsonReport.put(test.getFileName(), test.getJsonObject()); + } + + /** + * Returns a HashMap of (Tag, Value) pairs where value is the average test result of the photos tagged with that Tag + * @author Nicolò Cervo (g3) with the tutoring of Francesco Pham (g3) + */ + private HashMap getTagsStats() { + + HashMap tagStats = new HashMap<>(); //contains the cumulative score of every tag + HashMap tagOccurrences = new HashMap<>(); //contains the number of occurrences of each tag + + for(TestElement element : testElements) { + for (String tag : element.getTags()) { + if(tagStats.containsKey(tag)) { + float newValue = tagStats.get(tag) + element.getConfidence(); + tagStats.put(tag, newValue); + tagOccurrences.put(tag, tagOccurrences.get(tag) + 1); + }else{ + tagStats.put(tag, element.getConfidence()); + tagOccurrences.put(tag, 1); + } + } + } + + Log.i(TAG, "getTagStats():"); + for(String tag : tagStats.keySet()){ + tagStats.put(tag, tagStats.get(tag)/tagOccurrences.get(tag)); // average of the scores + Log.i(TAG, "-" + tag + " score: " + tagStats.get(tag)); + } + return tagStats; + } + + /** + * + * @return an HashMap: (Tag, Value) where value is the average gain result of the alterated photo tagged with that Tag + * @author Luca Moroldo - Credits: Nicolò Cerco (the structure of this method is the same of getTagsStats) + */ + private HashMap getAlterationsTagsGainStats() { + + //TODO think about: would it be better to get the stats for each tags collection rather than for each tag? + //for example: do we loose information if a photo is both rotated and cropped, and we don't consider that an extraordinary gain could be + //a consequence of this particular coupling of tags? + + HashMap alterationTagsGain = new HashMap<>(); //contains the % earning for each alteration tag + HashMap alterationTagsOccurrences = new HashMap<>(); //contains the occurrences of each tag + + for(TestElement element : testElements) { + //evaluate alterations if any + + + String[] alterationsNames = element.getAlterationsNames(); + if(alterationsNames != null) { + for(String alterationName : alterationsNames) { + for(String tag : element.getAlterationTags(alterationName)) { + Log.v(TAG, "AlterationTag " + tag); + if(alterationTagsGain.containsKey(tag)) { + float newGain = alterationTagsGain.get(tag) + (element.getAlterationConfidence(alterationName) - element.getConfidence()); + alterationTagsGain.put(tag, newGain); + alterationTagsOccurrences.put(tag, alterationTagsOccurrences.get(tag) + 1); + } else{ + alterationTagsGain.put(tag, element.getConfidence()); + alterationTagsOccurrences.put(tag, 1); + } + + } + } + } + } + for(String tag : alterationTagsGain.keySet()){ + alterationTagsGain.put(tag, alterationTagsGain.get(tag)/alterationTagsOccurrences.get(tag)); // average of the scores + Log.i(TAG, "-" + tag + " score: " + alterationTagsGain.get(tag)); + } + return alterationTagsGain; + } + + /** + * Convert statistics returned by getTagsStats() into a readable text + * @author Francesco Pham (g3) + */ + public String getTagsStatsString() { + HashMap tagsStats = getTagsStats(); + String report = "Average confidence by tags: \n"; + while(!tagsStats.isEmpty()){ + String keymin = getMinKey(tagsStats); + report = report + keymin + " : " + tagsStats.get(keymin) + "%\n"; + tagsStats.remove(keymin); + } + + HashMap alterationsTagsGainStats = getAlterationsTagsGainStats(); + report += "\nAvarage gain by alterations tags: \n"; + while(!alterationsTagsGainStats.isEmpty()) { + String keymin = getMinKey(alterationsTagsGainStats); + report = report + keymin + " : " + alterationsTagsGainStats.get(keymin) + "%\n"; + alterationsTagsGainStats.remove(keymin); + } + + Log.d(TAG, "Tag stats: \n" + report); + + return report; + + } + + /** + * Find and return key corresponding to minimum value + * @param map + * @return Key corresponding to minimum value + */ + private String getMinKey(Map map) { + String minKey = null; + float minValue = Float.MAX_VALUE; + for(String key : map.keySet()) { + float value = map.get(key); + if(value < minValue) { + minValue = value; + minKey = key; + } + } + return minKey; + } +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/ResultActivity.java b/app/src/main/java/unipd/se18/ocrcamera/ResultActivity.java index 627e5ce9..bb139115 100644 --- a/app/src/main/java/unipd/se18/ocrcamera/ResultActivity.java +++ b/app/src/main/java/unipd/se18/ocrcamera/ResultActivity.java @@ -1,14 +1,27 @@ package unipd.se18.ocrcamera; +import android.content.Context; +import android.content.Intent; +import android.content.SharedPreferences; import android.graphics.Bitmap; import android.graphics.BitmapFactory; -import android.support.annotation.Nullable; -import android.support.v7.app.AppCompatActivity; import android.os.Bundle; +import android.support.design.widget.FloatingActionButton; +import android.support.v7.app.AppCompatActivity; +import android.text.method.ScrollingMovementMethod; +import android.util.Log; +import android.view.Menu; +import android.view.MenuInflater; +import android.view.MenuItem; +import android.view.View; import android.widget.ImageView; import android.widget.TextView; -import java.io.File; +import unipd.se18.ocrcamera.recognizer.OCR; +import unipd.se18.ocrcamera.recognizer.OCRListener; +import unipd.se18.ocrcamera.recognizer.TextRecognizer; + +import static unipd.se18.ocrcamera.recognizer.TextRecognizer.getTextRecognizer; /** * Class used for showing the result of the OCR processing @@ -16,14 +29,36 @@ public class ResultActivity extends AppCompatActivity { /** - * The ImageView of the captured photo. + * The TextView of the extracted test from the captured photo. */ - private ImageView mImageView; + private TextView mOCRTextView; /** - * The TextView of the extracted test from the captured photo. + * Listener used by the extraction process to notify results */ - private TextView mOCRTextView; + private OCRListener textExtractionListener = new OCRListener() { + @Override + public void onTextRecognized(String text) { + /* + Text correctly recognized + -> prints it on the screen and saves it in the preferences + */ + mOCRTextView.setText(text); + saveTheResult(text); + } + + @Override + public void onTextRecognizedError(int code) { + /* + Text not correctly recognized + -> prints the error on the screen and saves it in the preferences + */ + String errorText = R.string.extraction_error + + " (" + R.string.error_code + code + ")"; + mOCRTextView.setText(errorText); + saveTheResult(errorText); + } + }; @Override protected void onCreate(Bundle savedInstanceState) { @@ -31,67 +66,104 @@ protected void onCreate(Bundle savedInstanceState) { setContentView(R.layout.activity_result); // UI components - mImageView = findViewById(R.id.img_captured_view); + ImageView mImageView = findViewById(R.id.img_captured_view); mOCRTextView = findViewById(R.id.ocr_text_view); + mOCRTextView.setMovementMethod(new ScrollingMovementMethod()); + FloatingActionButton fab = findViewById(R.id.newPictureFab); + fab.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + startActivity(new Intent(ResultActivity.this, CameraActivity.class)); + } + }); - //Retrieving captured image data and text from intent - String pathImage = getIntent().getStringExtra("imageDataPath"); - String OCRText = getIntent().getStringExtra("text"); - //Displaying the captured image to the user - displayImageFromByteArray(pathImage); + //Get image path and text of the last image from preferences + SharedPreferences prefs = getSharedPreferences("prefs", MODE_PRIVATE); + String pathImage = prefs.getString("imagePath", null); + String OCRText = prefs.getString("text", null); + + Bitmap lastPhoto = BitmapFactory.decodeFile(pathImage); + + if (lastPhoto != null) { + mImageView.setImageBitmap(Bitmap.createScaledBitmap(lastPhoto, lastPhoto.getWidth(), + lastPhoto.getHeight(), false)); + } else { + Log.e("ResultActivity", "error retrieving last photo"); + } //Displaying the text, from OCR or preferences if(OCRText != null) { - //Show the text of the last image - mOCRTextView.setText(OCRText); - } - else{ - //Utilization of OCR to retrieve text from the given image - extractTextFromImage(pathImage); - } + // Text in preferences + if(OCRText.equals("")) { + mOCRTextView.setText(R.string.no_text_found); + } else { + //Show the text of the last image + mOCRTextView.setText(OCRText); + } + } else { + // Views processing string + mOCRTextView.setText(R.string.processing); + + // Instance of an OCR recognizer + OCR ocrProcess = getTextRecognizer(TextRecognizer.Recognizer.mlKit, + textExtractionListener); - //Text shows when OCR are processing the image - mOCRTextView.setText(R.string.processing); + // Runs the operations of text extraction + ocrProcess.getTextFromImg(lastPhoto); + } } + /** - * Displays the captured image into UI given a specific byte array - * @param path A string that contains the path where is temporary saved the captured image. Not null. - * @modify mImageView The image view that is modified by the method - * @author Leonardo Rossi + * Saves the result obtained in the "prefs" preferences (Context.MODE_PRIVATE) + * - the name of the String is "text" + * @param text The text extracted by the process + * @author Pietro Prandini (g2) */ - private void displayImageFromByteArray(String path) { - File file = new File(path); - Bitmap bmp = BitmapFactory.decodeFile(file.getAbsolutePath()); - mImageView.setImageBitmap(Bitmap.createScaledBitmap(bmp, 300, 300, false)); + private void saveTheResult(String text) { + // Saving in the preferences + SharedPreferences sharedPref = getApplicationContext().getSharedPreferences("prefs", + Context.MODE_PRIVATE); + SharedPreferences.Editor editor = sharedPref.edit(); + editor.putString("text", text); + editor.apply(); } /** - * Retrieves the text from the given byte array - * @param path A string that contains the path where is temporary saved the captured image. Not null. - * @modify mOCRTextView It will contains the text extracts from the image - * @author Leonardo Rossi + * Menu inflater + * @author Francesco Pham */ - private void extractTextFromImage(String path) { - //Converting byte array into bitmap - File file = new File(path); - Bitmap bmp = BitmapFactory.decodeFile(file.getAbsolutePath()); - //Call to text extractor method to get the text from the given image - TextExtractor extractor = new TextExtractor(this); - extractor.getTextFromImg(bmp); - //Definition of the observer which will be responsible of updating the UI once the text extractor has finished its work - android.arch.lifecycle.Observer obsText = new android.arch.lifecycle.Observer() { - @Override - public void onChanged(@Nullable String s) { - if(s != null && s.equals("")) { - mOCRTextView.setText(R.string.no_text_found); - } else if (s != null) { - mOCRTextView.setText(s.toUpperCase()); - } - } - }; - extractor.extractedText.observe(this, obsText); + @Override + public boolean onCreateOptionsMenu(Menu menu) { + MenuInflater inflater = getMenuInflater(); + inflater.inflate(R.menu.result_menu, menu); + return true; } + /** + * Handling click events on the menu + * @author Francesco Pham - modified by Stefano Romanello + */ + @Override + public boolean onOptionsItemSelected(MenuItem item) { + // Handle item selection + switch (item.getItemId()) { + case R.id.test: + Intent i = new Intent(ResultActivity.this, TestsListActivity.class); + startActivity(i); + return true; + case R.id.download_photos: + Intent download_intent = new Intent(ResultActivity.this, + DownloadDbActivity.class); + startActivity(download_intent); + return true; + case R.id.manual_test: + Intent manualTest= new Intent(ResultActivity.this, ManualTestOnSinglePhoto.class); + startActivity(manualTest); + default: + return super.onOptionsItemSelected(item); + } + } } + diff --git a/app/src/main/java/unipd/se18/ocrcamera/TestDetailsActivity.java b/app/src/main/java/unipd/se18/ocrcamera/TestDetailsActivity.java new file mode 100644 index 00000000..f53ed7cb --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/TestDetailsActivity.java @@ -0,0 +1,367 @@ +package unipd.se18.ocrcamera; + +import android.content.Context; +import android.content.Intent; +import android.graphics.Bitmap; +import android.graphics.Color; +import android.graphics.Typeface; +import android.support.v7.app.AppCompatActivity; +import android.os.Bundle; +import android.util.DisplayMetrics; +import android.view.Display; +import android.view.View; +import android.view.WindowManager; +import android.widget.ImageView; +import android.widget.RelativeLayout; +import android.widget.TextView; + +import java.text.DecimalFormat; + +/** + * Prints to the screen the details of a TestElement + * @author Pietro Prandini (g2) + */ +public class TestDetailsActivity extends AppCompatActivity { + /** + * String used for the log of this class + */ + private String TAG = "TestDetailsActivity -> "; + + /** + * TestElement to be viewed + */ + private TestElement entry; + + /* + Limits for choosing the color of the correctness relatively to the goodness of the extraction + */ + private float redUntil; + private float yellowUntil; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_test_element_details); + + // Retrieves the TestElement to be viewed (default == 0) + Intent lastIntent = getIntent(); + int position = lastIntent.getIntExtra(TestsListAdapter.positionString,0); + entry = TestsListAdapter.getTestElements()[position]; + + // Retrieves the limits for the correctness color (default == 0) + redUntil = lastIntent.getIntExtra(TestsListAdapter.redUntilString, 0); + yellowUntil = lastIntent.getIntExtra(TestsListAdapter.yellowUntilString, 0); + + // Sets the correctness value + TextView correctness = findViewById(R.id.correctness_view); + float confidence = entry.getConfidence(); + correctness.setText(formatPercentString(confidence)); + + // Sets the color of the correctness text value + correctness.setTextColor(chooseColorOfValue(confidence,redUntil,yellowUntil)); + + // Sets the name of the pic + TextView name = findViewById(R.id.pic_name_view); + String picName = entry.getFileName(); + name.setText(picName); + + // Sets the pic view + ImageView analyzedPic = findViewById(R.id.pic_view); + String imagePath = entry.getImagePath(); + Bitmap img = Utils.loadBitmapFromFile(imagePath); + analyzedPic.setImageBitmap(scaleBitmap(TestDetailsActivity.this, img)); + + // Sets the Tags text + TextView tags = findViewById(R.id.tags_view); + StringBuilder assignedTags = new StringBuilder(); + for(String tag: entry.getTags()) { + assignedTags.append(tag).append(", "); + } + tags.setText(assignedTags.toString()); + + // Sets the ingredients text + TextView ingredients = findViewById(R.id.ingredients_view); + StringBuilder realIngredients = new StringBuilder(); + for(String ingredient: entry.getIngredientsArray()) { + realIngredients.append(ingredient).append(", "); + } + ingredients.setText(realIngredients); + + // Sets the extracted text + TextView extractedText = findViewById(R.id.extractedText_view); + extractedText.setText(entry.getRecognizedText()); + + // Sets the notes text + TextView notes = findViewById(R.id.notes_view); + notes.setText(entry.getNotes()); + + // Sets the alterations view + setAlterationsView( + TestDetailsActivity.this, + (RelativeLayout) findViewById(R.id.result_view), + R.id.notes_view, + entry, + true); + } + + /** + * Formats the percent String + * @param value The percent value + * @return The String formatted + * @author Pietro Prandini (g2) + */ + protected static String formatPercentString(float value) { + return new DecimalFormat("#0").format(value) + " %"; + } + + /** + * Return the appropriate color of a value. + * The color is picket between red (not good), yellow (ok) and green (very good). + * @param value The value to be colored + * @param redUntil Under this value would be red and upper this yellow. + * @param yellowUntil Under this value would be yellow and upper this green. + * @return The relative color. + * @author Pietro Prandini (g2) + */ + protected static int chooseColorOfValue(float value, float redUntil, float yellowUntil) { + if(value < redUntil) { + return Color.RED; + } else if (value < yellowUntil) { + return Color.YELLOW; + } else { + return Color.GREEN; + } + } + + /** + * Return the appropriate color of a value. + * The color is picket between red (not better than redUntil) and green (better than redUntil). + * @param value The value to be colored + * @param redUntil Under this value would be red and upper this yellow. + * @return The relative color. + * @author Pietro Prandini (g2) + */ + protected static int chooseColorOfValue(float value, float redUntil) { + if(value < redUntil) { + return Color.RED; + } else { + return Color.GREEN; + } + } + + /** + * Scales a Bitmap pic relatively to the width of the screen + * @param context The context of the activity + * @param img The Bitmap to be scaled + * @return Bitmap scaled with the width same as the width of the screen + * @author Pietro Prandini (g2) + */ + protected static Bitmap scaleBitmap(Context context, Bitmap img) { + // Obtains the original dimensions of the pic + int imgWidth = img.getWidth(); + int imgHeight = img.getHeight(); + + // Obtains the metrics of the screen (useful to obtain the with of the screen) + WindowManager mWindowManager = + (WindowManager) context.getSystemService(Context.WINDOW_SERVICE); + DisplayMetrics mDisplayMetrics = new DisplayMetrics(); + Display mDisplay = mWindowManager.getDefaultDisplay(); + mDisplay.getMetrics(mDisplayMetrics); + + // Calculates the new dimensions of the pic + int scaledWidth = mDisplayMetrics.widthPixels; + int scaledHeight = (scaledWidth*imgHeight)/imgWidth; + + // Returns the Bitmap scaled + return Bitmap.createScaledBitmap(img, scaledWidth, scaledHeight,false); + } + + /** + * Set the alterations text to the view of the activity + * @param context The context where would be the alterations text + * @param relativeLayout The layout to add the text views + * @param idBelowOf The id of the view where putting the alterations text below of + * @param element The test element where searching the alterations + * @param viewDetails True for viewing details of alterations, false otherwise + * @author Pietro Prandini (g2) + */ + protected static void setAlterationsView(Context context, RelativeLayout relativeLayout, + int idBelowOf, TestElement element, Boolean viewDetails) { + String[] alterations = element.getAlterationsNames(); + StringBuilder alterationsText = new StringBuilder(); + if(alterations != null) { + // Sets details of alterations + for (String alteration : alterations) { + // Prepares the alterations String + float confidenceOfAlteration = element.getAlterationConfidence(alteration); + alterationsText.append(alteration) + .append(" - confidence ") + .append(TestDetailsActivity.formatPercentString(confidenceOfAlteration)) + .append("\n"); + + // Prepares the alterations TextView + TextView alterationsView = new TextView(context); + alterationsView.setText(alterationsText.toString()); + alterationsView.setTextColor( + chooseColorOfValue(confidenceOfAlteration, element.getConfidence())); + // Sets shadow (supports the coloured view) + float radius = 1; + float dx = 0; + float dy = 0; + alterationsView.setShadowLayer(radius,dx,dy,Color.BLACK); + + // Sets bold (supports the coloured view) + alterationsView.setTypeface(null, Typeface.BOLD); + + // Padding + int padding = 10; + alterationsView.setPadding(padding, padding, padding, padding); + + // Sets layout params + RelativeLayout.LayoutParams paramsView = new RelativeLayout.LayoutParams( + RelativeLayout.LayoutParams.WRAP_CONTENT, + RelativeLayout.LayoutParams.WRAP_CONTENT); + + // View horizontally centered + paramsView.addRule(RelativeLayout.CENTER_HORIZONTAL); + + // Adds the view + idBelowOf = addViewBelow(relativeLayout,paramsView,idBelowOf,alterationsView); + + // Sets details if required + if(viewDetails) { + idBelowOf = viewAlterationDetails( + context, + relativeLayout, + idBelowOf, + element, + alteration + ); + } + } + } + } + + /** + * Sets the details of an altered test + * @param context The context where would be the alterations text + * @param relativeLayout The layout to add the text views + * @param idBelowOf The id of the view where putting the alterations text below of + * @param element The test element where searching the alterations + * @param alteration The alteration type String + * @author Pietro Prandini (g2) + */ + protected static int viewAlterationDetails(Context context, RelativeLayout relativeLayout, + int idBelowOf, TestElement element, String alteration) { + + // Obtains the altered pic + String imagePath = element.getAlterationImagePath(alteration); + Bitmap img = scaleBitmap(context,Utils.loadBitmapFromFile(imagePath)); + ImageView picView = new ImageView(context); + picView.setImageBitmap(img); + + // Adds the pics view (set the id to the id of that view) + idBelowOf = addViewBelow(relativeLayout, idBelowOf, picView); + + // Adds the alteration details + // Tags title + TextView tagsTitle = new TextView(context); + tagsTitle.setText(R.string.tags); + + int titlePadding = 5; + tagsTitle.setPadding(titlePadding,titlePadding,titlePadding,titlePadding); + tagsTitle.setTypeface(Typeface.DEFAULT_BOLD); + + idBelowOf = addViewBelow(relativeLayout,idBelowOf,tagsTitle); + + // Tags details + TextView tags = new TextView(context); + int detailsPadding = 10; + // Sets the Tags text + StringBuilder assignedTags = new StringBuilder(); + for(String tag: element.getAlterationTags(alteration)) { + assignedTags.append(tag).append(", "); + } + + tags.setText(assignedTags.toString()); + tags.setPadding(detailsPadding, detailsPadding, detailsPadding,detailsPadding); + + idBelowOf = addViewBelow(relativeLayout,idBelowOf,tags); + + // Extracted text title + TextView extractedTextTitle = new TextView(context); + extractedTextTitle.setText(R.string.extrected_text); + + extractedTextTitle.setPadding(titlePadding,titlePadding,titlePadding,titlePadding); + extractedTextTitle.setTypeface(Typeface.DEFAULT_BOLD); + + idBelowOf = addViewBelow(relativeLayout,idBelowOf,extractedTextTitle); + + // Extracted text details + TextView extractedText = new TextView(context); + extractedText.setText(element.getAlterationRecognizedText(alteration)); + extractedText.setPadding(detailsPadding, detailsPadding, detailsPadding, detailsPadding); + + idBelowOf = addViewBelow(relativeLayout,idBelowOf,extractedText); + + // Notes title + TextView notesTitle = new TextView(context); + notesTitle.setText(R.string.notes); + + notesTitle.setPadding(titlePadding,titlePadding,titlePadding,titlePadding); + notesTitle.setTypeface(Typeface.DEFAULT_BOLD); + + idBelowOf = addViewBelow(relativeLayout,idBelowOf,notesTitle); + + // Notes details + TextView notes = new TextView(context); + notes.setText(element.getAlterationNotes(alteration)); + notes.setPadding(detailsPadding, detailsPadding, detailsPadding, detailsPadding); + + idBelowOf = addViewBelow(relativeLayout,idBelowOf,notes); + + return idBelowOf; + } + + /** + * Adds a view below with parameters set to "WRAP_CONTENT" + * @param relativeLayout The layout to add the text views + * @param idBelowOf The id of the view where putting the alterations text below of + * @param view The view to be added + * @return The id of the view added + * @author Pietro Prandini (g2) + */ + protected static int addViewBelow(RelativeLayout relativeLayout, + int idBelowOf, View view) { + // Configures the layout parameters + RelativeLayout.LayoutParams paramsView = new RelativeLayout.LayoutParams( + RelativeLayout.LayoutParams.WRAP_CONTENT, + RelativeLayout.LayoutParams.WRAP_CONTENT); + + return addViewBelow(relativeLayout, paramsView, idBelowOf,view); + } + + /** + * Adds a view below + * @param relativeLayout The layout to add the text views + * @param layoutParams The parameters of the layout + * @param idBelowOf The id of the view where putting the alterations text below of + * @param view The view to be added + * @return The id of the view added + * @author Pietro Prandini (g2) + */ + protected static int addViewBelow(RelativeLayout relativeLayout, + RelativeLayout.LayoutParams layoutParams, + int idBelowOf, View view) { + // Puts below of the id passed + layoutParams.addRule(RelativeLayout.BELOW, idBelowOf); + + // Adds the view + relativeLayout.addView(view,layoutParams); + + // Sets the appropriate id + view.setId(View.generateViewId()); + idBelowOf = view.getId(); + return idBelowOf; + } +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/TestElement.java b/app/src/main/java/unipd/se18/ocrcamera/TestElement.java new file mode 100644 index 00000000..0f7aee22 --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/TestElement.java @@ -0,0 +1,371 @@ +package unipd.se18.ocrcamera; + +import android.util.Log; + +import org.json.JSONException; +import org.json.JSONObject; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; + +/** + * Class that contains a single test element, used in PhotoTester to build a single test and on + * TestsListAdapter to show data in a listview. + * @author Luca Moroldo, Francesco Pham + */ + +public class TestElement { + + private static final String TAG = "TestElement"; + + private static final String NOTES_KEY = "notes"; + private static final String EXTRACTED_TEST_KEY = "extracted_text"; + private static final String TAGS_KEY = "tags"; + private static final String INGREDIENTS_KEY = "ingredients"; + private static final String ALTERATIONS_KEY = "alterations"; + private static final String CONFIDENCE_KEY = "confidence"; + + private String imagePath; + private JSONObject jsonObject; + private String fileName; + private HashMap alterationsImagesPath; + + /** + * + * @param imagePath String containing the path to the image associated with the jsonObject + * @param jsonObject JSONObject containing test data (ingredients, tags, notes, alterations if any) + * @param fileName name of the test + * @author Luca Moroldo - g3 + */ + public TestElement(String imagePath, JSONObject jsonObject, String fileName) { + this.imagePath = imagePath; + this.jsonObject = jsonObject; + this.fileName = fileName; + //prepare alterations images if there is any + String[] alterationsNames = getAlterationsNames(); + if(alterationsNames != null) { + alterationsImagesPath = new HashMap<>(); + for(String alterationName : alterationsNames) { + alterationsImagesPath.put(alterationName, null); + } + } + } + + /** + * Array of Strings, each string is an ingredient, ingredients are separated on comma + * @return Array of strings, each string is an ingredient + */ + public String[] getIngredientsArray() { + String ingredients = getIngredients(); + String[] ingredientsArr = ingredients.trim().split("\\s*,\\s*"); //split removing whitespaces + return ingredientsArr; + } + + /** + * @return ingredients text of this test if exist, null otherwise + * @author Luca Moroldo - g3 + */ + public String getIngredients() { + try { + return jsonObject.getString(INGREDIENTS_KEY); + } catch (JSONException e) { + Log.i(TAG, "Failed to get ingredients inside test: " + fileName); + } + return null; + } + + /** + * @return array of tags of this test if exist, null otherwise + * @author Luca Moroldo - g3 + */ + public String[] getTags() { + try { + return Utils.getStringArrayFromJSON(jsonObject, TAGS_KEY); + } catch (JSONException e) { + Log.i(TAG, "Failed to get tags inside test: " + fileName); + } + return null; + } + + /** + * @return String with the path to the image associated with this test, can be null if it hasn't + * been set + */ + public String getImagePath() { + return imagePath; + } + + /** + * @return name of this test + */ + public String getFileName() { + return fileName; + } + + /** + * @return notes text of this test if exist, null otherwise + * @author Luca Moroldo - g3 + */ + public String getNotes() { + try { + return jsonObject.getString(NOTES_KEY); + } catch (JSONException e) { + Log.i(TAG, "Failed to get notes inside test: " + fileName); + } + return null; + } + + /** + * Getter for alterations filenames of a test (e.g. cropped photo) if any + * @return array of strings if the element has any alteration (each string is a filename), null otherwise + * @author Luca Moroldo - g3 + */ + public String[] getAlterationsNames() { + + JSONObject alterations = null; + try { + alterations = jsonObject.getJSONObject(ALTERATIONS_KEY); + } catch (JSONException e) { + Log.i(TAG, "There are no alterations inside test: " + fileName); + return null; + } + + ArrayList alterationsNames= new ArrayList(); + + Iterator keys = alterations.keys(); + + while(keys.hasNext()) { + String key = keys.next(); + alterationsNames.add(key); + } + + return alterationsNames.toArray(new String[0]); + } + + /** + * @return Float confidence of this test if present, -1 otherwise + * @author Luca Moroldo - g3 + */ + public float getConfidence() { + try { + String confidence = jsonObject.getString(CONFIDENCE_KEY); + return Float.parseFloat(confidence); + } catch (JSONException e) { + Log.i(TAG, "Failed to get confidence in test: " + fileName); + } + return -1; + } + + /** + * @return String recognized text of this test if present, null otherwise + * @author Luca Moroldo - g3 + */ + public String getRecognizedText() { + try { + return jsonObject.getString(EXTRACTED_TEST_KEY); + } catch (JSONException e) { + Log.i(TAG, "Failed to get recognized text in test: " + fileName); + } + return null; + } + + /** + * Get an alteration extracted text + * @param alterationName name of an existing alteration inside this test + * @return alteration recognized text if it's set, null if recognized text hasn't been set or if there isn't any alteration named with the given param + * @author Luca Moroldo - g3 + */ + public String getAlterationRecognizedText(String alterationName) { + JSONObject jsonAlteration = getAlterationWithName(alterationName); + if(jsonAlteration != null) { + try { + return jsonAlteration.getString(EXTRACTED_TEST_KEY); + } catch (JSONException e) { + Log.i(TAG, "Failed to get recognized text in alteration: " + alterationName + " inside test: " + fileName); + } + } + return null; + } + + /** + * Get an alteration confidence + * @param alterationName name of an existing alteration inside this test + * @return confidence if it has been set, -1 if the confidence hasn't been set or if there isn't any alteration named with the given param + * @author Luca Moroldo - g3 + */ + public float getAlterationConfidence(String alterationName) { + + JSONObject jsonAlteration = getAlterationWithName(alterationName); + if(jsonAlteration != null) { + try { + String confidence = jsonAlteration.getString(CONFIDENCE_KEY); + return Float.parseFloat(confidence); + } catch (JSONException e) { + Log.i(TAG, "Failed to get confidence in alteration: " + alterationName + " inside test: " + fileName); + } + } + return -1; + } + + /** + * Get an alteration associated image + * @param alterationName name of an existing alteration inside this test + * @return String with the path to the image associated with the test if it has been set, + * null if the image hasn't been set or if there isn't any alteration named with the given param + * @author Luca Moroldo - g3 + */ + public String getAlterationImagePath(String alterationName) { + if(alterationsImagesPath.containsKey(alterationName)) + return alterationsImagesPath.get(alterationName); + else + Log.i(TAG, "No image path set for alteration " + alterationName + " in test " + fileName); + return null; + } + + /** + * @param alterationName alterationName name of an existing alteration inside this test + * @return notes text, null if there isn't any alteration named with the given param or if there aren't notes in this test + * @author Luca Moroldo - g3 + */ + public String getAlterationNotes(String alterationName) { + JSONObject jsonAlteration = getAlterationWithName(alterationName); + if(jsonAlteration != null) { + try { + return jsonAlteration.getString(NOTES_KEY); + } catch (JSONException e) { + Log.i(TAG, "Failed to get notes from alteration: " + alterationName + " inside test: " + fileName); + } + } + return null; + } + + /** + * @param alterationName alterationName name of an existing alteration inside this test + * @return tags array, null if there isn't any alteration named with the given param + * @author Luca Moroldo - g3 + */ + public String[] getAlterationTags(String alterationName) { + + JSONObject jsonAlteration = getAlterationWithName(alterationName); + if(jsonAlteration != null) { + try { + return Utils.getStringArrayFromJSON(jsonAlteration, TAGS_KEY); + } catch (JSONException e) { + Log.i(TAG, "Failed to get tags from altaration: " + alterationName + " inside test: " + fileName); + } + } + return null; + } + + /** + * @return JSONObject associated to this test + */ + public JSONObject getJsonObject() { return jsonObject; } + + /** + * @param confidence Float that will be associated to this test with key 'confidence' + * @modify jsonObject of this TestElement + */ + public void setConfidence(float confidence) { + try { + jsonObject.put(CONFIDENCE_KEY, Float.toString(confidence)); + } catch (JSONException e) { + Log.i(TAG, "Failed to set confidence in test " + fileName); + } + } + + /** + * @param text String that will be set in this test with key 'extracted_text' + * @modify jsonObject of this TestElement + * @author Luca Moroldo - g3 + */ + public void setRecognizedText(String text) { + try { + jsonObject.put(EXTRACTED_TEST_KEY, text); + } catch (JSONException e) { + Log.i(TAG, "Failed to set recognized text in test " + fileName); + } + } + + /** + * associate a image path to an alteration of this test + * @param alterationName name of an existing alteration inside this test + * @param imagePath String with the path to the image related to the alteration test + * @modify jsonObject of this TestElement + * @return true if image was set correctly, false if alteration name doesn't exist + * @author Luca Moroldo - g3 + */ + public boolean setAlterationImagePath(String alterationName, String imagePath) { + if(alterationsImagesPath.containsKey(alterationName)) { + alterationsImagesPath.put(alterationName, imagePath); + return true; + } + Log.i(TAG, "No alteration found in " + fileName + " with name " + alterationName); + return false; + } + + /** + * Associate a recognized text to the alteration inside this test, if present + * @param alterationName alterationName name of an existing alteration inside this test + * @param text recognized text of the alteration that will be set + * @modify jsonObject of this TestElement + * @author Luca Moroldo - g3 + */ + public void setAlterationRecognizedText(String alterationName, String text) { + + JSONObject jsonAlteration = getAlterationWithName(alterationName); + if(jsonAlteration != null) { + try { + jsonAlteration.put(EXTRACTED_TEST_KEY, text); + } catch (JSONException e) { + Log.i(TAG, "Failed to set confidence of alteration: " + alterationName + " inside test: " + fileName); + } + } + } + + /** + * @param alterationName alterationName name of an existing alteration inside this test + * @param alterationConfidence value of the confidence of the alteration that will be set + * @modify jsonObject of this TestElement + * @author Luca Moroldo - g3 + */ + public void setAlterationConfidence(String alterationName, float alterationConfidence) { + + JSONObject jsonAlteration = getAlterationWithName(alterationName); + if(jsonAlteration != null) { + try { + jsonAlteration.put(CONFIDENCE_KEY, Float.toString(alterationConfidence)); + } catch (JSONException e) { + Log.i(TAG, "Failed to set confidence of alteration: " + alterationName + " inside test: " + fileName); + } + } + } + + @Override + public String toString() { return jsonObject.toString(); } + + + /** + * Get from JSON test data an alteration JSONObject with the name given as argument + * @param alterationName name of an alteration inside this test + * @return JSONObject associated with the alteration name given as argument, null if it doesn't exist + */ + private JSONObject getAlterationWithName(String alterationName) { + JSONObject jsonAlterations = null; + try { + jsonAlterations = jsonObject.getJSONObject(ALTERATIONS_KEY); + } catch (JSONException e) { + Log.i(TAG, "No alteration found in " + fileName ); + return null; + } + + try { + JSONObject jsonAlteration = jsonAlterations.getJSONObject(alterationName); + return jsonAlteration; + } catch (JSONException e) { + Log.i(TAG, "There is no alteration with name " + alterationName + " inside test " + fileName); + return null; + } + } +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/TestListener.java b/app/src/main/java/unipd/se18/ocrcamera/TestListener.java new file mode 100644 index 00000000..b2b31635 --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/TestListener.java @@ -0,0 +1,6 @@ +package unipd.se18.ocrcamera; + +public interface TestListener { + void onTestFinished(); + void onAlterationAnalyzed(); +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/TestsListActivity.java b/app/src/main/java/unipd/se18/ocrcamera/TestsListActivity.java new file mode 100644 index 00000000..3d4f19ce --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/TestsListActivity.java @@ -0,0 +1,238 @@ +package unipd.se18.ocrcamera; + +import android.Manifest; +import android.annotation.SuppressLint; +import android.content.Context; +import android.content.Intent; +import android.content.pm.PackageManager; +import android.os.AsyncTask; +import android.os.Bundle; +import android.support.annotation.NonNull; +import android.support.v4.app.ActivityCompat; +import android.support.v7.app.AppCompatActivity; +import android.widget.ListView; +import android.widget.ProgressBar; +import android.widget.TextView; +import android.widget.Toast; + +/** + * Activity for showing the result of the tests + * Pietro Prandini (g2) + */ +public class TestsListActivity extends AppCompatActivity { + /** + * String used for the logs of this class + */ + private static final String TAG = "TestsListActivity"; + + /** + * The custom request code requested for permission use + */ + private static final int MY_READ_EXTERNAL_STORAGE_REQUEST_CODE = 300; + + /** + * Prepares the activity to show the test results. + * More details at: {@link ActivityCompat#checkSelfPermission(Context, String)} + * @param savedInstanceState Bundle of the last instance state of the app + */ + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + // Sets the layout + setContentView(R.layout.activity_test_result); + + // Checks the permissions + if (ActivityCompat.checkSelfPermission(this, + Manifest.permission.WRITE_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) { + String[] permissions = { Manifest.permission.WRITE_EXTERNAL_STORAGE }; + ActivityCompat.requestPermissions(this, permissions, + MY_READ_EXTERNAL_STORAGE_REQUEST_CODE); + return; + } + + // Sets the view of the list + ListView listEntriesView = findViewById(R.id.test_entries_list); + + // Sets the elements of the list as AsyncTask + AsyncReport report = new AsyncReport(listEntriesView); + report.execute(); + } + + /** + * Execute the ocr task for every test pic in the storage + * TraPietro Prandini (g2) + */ + @SuppressLint("StaticFieldLeak") + private class AsyncReport extends AsyncTask { + /** + * The ListView where showing the results. + */ + private ListView listEntriesView; + + /** + * The String of the test pics directory path. + */ + private String dirPath; + + /** + * The String of the message to show when the task is in progress + */ + private String progressMessage; + + /** + * Instance of PhotoTester used for doing the tests + */ + private PhotoTester tester; + + /** + * The String where will be stored the report + */ + private String report; + + /** + * The progress bar used for indicating the progress of the tests + */ + private ProgressBar progressBar; + + /** + * The text used for indicating the progress of the tests + */ + private TextView progressText; + + /** + * The number of tested elements so far + */ + private int testedElements = 0; + /** + * Constructor of the class + * @param listEntriesView The ListView used for showing the results as list + */ + AsyncReport(ListView listEntriesView) { + this.listEntriesView = listEntriesView; + this.dirPath = PhotoDownloadTask.PHOTOS_FOLDER; + } + + /** + * Prepares the task to start + * More details at: {@link AsyncTask#onPreExecute()} + */ + @Override + protected void onPreExecute() { + progressBar = findViewById(R.id.tests_progress_bar); + progressText = findViewById(R.id.progress_testing_text); + } + + /** + * Does the task in background - Does the tests + * More details at: {@link AsyncTask#doInBackground(Object[])} + * @param voids objects received during the execution + * @modify tester Instance of PhotoTester used for doing the tests + * @modify listEntriesView The ListView where showing the results. + * @modify report The String where will be stored the report + * @return the result of the objects processed + */ + @Override + protected Void doInBackground(Void... voids) { + this.tester = new PhotoTester(dirPath); + progressBar.setMax(tester.getTestSize()); + + // Listener useful for updating the progress bar + TestListener testListener = new TestListener() { + @Override + public void onTestFinished() { + // +1 test finished -> +1 progress bar + publishProgress(++testedElements); + } + + @Override + public void onAlterationAnalyzed() { + // not useful in this case + } + }; + tester.setTestListener(testListener); + + // publishes progress + try { + report = tester.testAndReport(); + } catch (InterruptedException e) { + e.printStackTrace(); + report = "Elaboration interrupted"; + } + + runOnUiThread(new Runnable() { + @Override + public void run() { + TestsListAdapter adapter = + new TestsListAdapter( + TestsListActivity.this, + tester.getTestElements() + ); + listEntriesView.setAdapter(adapter); + } + }); + + return null; + } + + @Override + protected void onProgressUpdate(Integer... values) { + progressBar.setProgress(testedElements); + String progress = "Tested: " + values[0] + + " of " + tester.getTestSize(); + progressText.setText(progress); + } + + /** + * End of the task + * More details at: {@link AsyncTask#onPostExecute(Object)} + * @param v The object returned by the processing + * @modify progressDialog The ProgressDialog object used while the AsyncTask is running + * @modify listEntriesView The ListView where showing the results. + */ + @Override + protected void onPostExecute(Void v) { + //add statistics author: Francesco Pham + TextView statsView = new TextView(TestsListActivity.this); + String statsText = ""; + + statsText = tester.getTagsStatsString(); + + statsView.setText(statsText); + listEntriesView.addHeaderView(statsView); + } + } + + /** + * Catches and controls the response of the permissions request + * More details at: {@link Intent}, {@link Manifest.permission}, + * {@link AppCompatActivity#onRequestPermissionsResult(int, String[], int[])} + * @param requestCode The code assigned to the request + * @param permissions The list of the permissions requested + * @param grantResults The results of the requests + * @author Pietro Prandini (g2) + */ + @Override + public void onRequestPermissionsResult(int requestCode, @NonNull String permissions[], + @NonNull int[] grantResults) { + switch (requestCode) { + case MY_READ_EXTERNAL_STORAGE_REQUEST_CODE: { + if(grantResults.length == 0 || + grantResults[0] != PackageManager.PERMISSION_GRANTED) { + // Permissions is not granted + // notifies it by a toast + runOnUiThread(new Runnable() { + @Override + public void run() { + Toast.makeText(TestsListActivity.this, + R.string.permissions_not_granted, Toast.LENGTH_LONG).show(); + } + }); + // Destroy the activity + finish(); + } + } + } + + } +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/TestsListAdapter.java b/app/src/main/java/unipd/se18/ocrcamera/TestsListAdapter.java new file mode 100644 index 00000000..5a3d12e9 --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/TestsListAdapter.java @@ -0,0 +1,136 @@ +package unipd.se18.ocrcamera; + +import android.content.Context; +import android.content.Intent; +import android.view.LayoutInflater; +import android.view.View; +import android.view.ViewGroup; +import android.widget.BaseAdapter; +import android.widget.Button; +import android.widget.RelativeLayout; +import android.widget.TextView; + +/** + * Adapter for the view of the processing result of the pics + * @author Pietro Prandini (g2) + */ +public class TestsListAdapter extends BaseAdapter { + /** + * String used for the logs of this class + */ + private final String TAG = "TestsListAdapter -> "; + + /** + * Context of the app + */ + private Context context; + + /** + * Elements of test + */ + private static TestElement[] entries; + + /* + Limits for choosing the color of the correctness relatively to the goodness of the extraction + */ + private float redUntil; + private float yellowUntil; + + /* + Strings used for passing by intent some data to the other activity + */ + static final String positionString = "position"; + static final String redUntilString = "redUntil"; + static final String yellowUntilString = "yellowUntil"; + + /** + * Defines an object of AdapterTestElement type + * @param context The reference to the activity where the adapter will be used + * @param entries The list of the test elements containing data from photos test + */ + TestsListAdapter(Context context, TestElement[] entries) { + this.context = context; + TestsListAdapter.entries = entries; + } + + @Override + public int getCount() { return entries.length; } + + @Override + public Object getItem(int position) { return entries[position]; } + + @Override + public long getItemId(int position) { + // The prefix is "foto", so the suffix starts at 4 + int suffix = 4; + return Integer.parseInt(entries[position].getFileName().substring(suffix)); + } + + @Override + public View getView(final int position, View convertView, ViewGroup parent) { + if (convertView == null) { + convertView = LayoutInflater.from(context).inflate(R.layout.test_element, parent, false); + } + + // Set the correctness value + TextView correctness = convertView.findViewById(R.id.correctness_view); + float confidence = entries[position].getConfidence(); + correctness.setText(TestDetailsActivity.formatPercentString(confidence)); + + // Set the color of the correctness text value + redUntil = 70; + yellowUntil = 85; + correctness.setTextColor( + TestDetailsActivity.chooseColorOfValue(confidence, redUntil, yellowUntil) + ); + + // Set the name of the pic + TextView name = convertView.findViewById(R.id.pic_name_view); + String picName = entries[position].getFileName(); + name.setText(picName); + + // Set the Tags text + TextView tags = convertView.findViewById(R.id.tags_view); + StringBuilder assignedTags = new StringBuilder(); + for (String tag : entries[position].getTags()) { + assignedTags.append(tag).append(", "); + } + tags.setText(assignedTags.toString()); + + // Set alterations view + TestDetailsActivity.setAlterationsView( + context, + (RelativeLayout) convertView.findViewById(R.id.result_view), + R.id.tags_view, + entries[position], + false + ); + + Button viewDetailsButton = convertView.findViewById(R.id.view_details_button); + viewDetailsButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View v) { + // Prepares the Intent for launching the TestDetailsActivity + Intent testDetailsActivity = new Intent(context, TestDetailsActivity.class); + + // Passes some values by the intents + testDetailsActivity.putExtra(positionString,position); + testDetailsActivity.putExtra(redUntilString,redUntil); + testDetailsActivity.putExtra(yellowUntilString,yellowUntil); + + // Starts the activity + context.startActivity(testDetailsActivity); + } + }); + return convertView; + } + + /** + * Gets the test elements analyzed + * @return The array of the Test Elements analyzed + * @author Pietro Prandini (g2) + */ + static TestElement[] getTestElements() { + return TestsListAdapter.entries; + } +} \ No newline at end of file diff --git a/app/src/main/java/unipd/se18/ocrcamera/TextExtractor.java b/app/src/main/java/unipd/se18/ocrcamera/TextExtractor.java deleted file mode 100644 index 1e96505a..00000000 --- a/app/src/main/java/unipd/se18/ocrcamera/TextExtractor.java +++ /dev/null @@ -1,75 +0,0 @@ -package unipd.se18.ocrcamera; - -import android.arch.lifecycle.MutableLiveData; -import android.content.Context; -import android.content.SharedPreferences; -import android.graphics.Bitmap; -import android.preference.PreferenceManager; -import android.support.annotation.NonNull; - -import com.google.android.gms.tasks.OnFailureListener; -import com.google.android.gms.tasks.OnSuccessListener; -import com.google.firebase.ml.vision.FirebaseVision; -import com.google.firebase.ml.vision.common.FirebaseVisionImage; -import com.google.firebase.ml.vision.text.FirebaseVisionText; -import com.google.firebase.ml.vision.text.FirebaseVisionTextRecognizer; - -/** - * Class the implements the common OCR wrapper to retrieve text from an image - * @author Leonardo Rossi - */ -public class TextExtractor implements OCRWrapper { - MutableLiveData extractedText; - Context context; - - /** - * It defines an object of type TextExtractor - */ - public TextExtractor(Context context) { - extractedText = new MutableLiveData<>(); - this.context = context; - } - - /** - * Extracts a text from a given image. - * @param img The image in a Bitmap format - * @return The String of the text recognized (empty String if nothing is recognized) - * @author Leonardo Rossi - */ - @Override - public String getTextFromImg(Bitmap img) { - //Defines the image that will be analysed to get the text - FirebaseVisionImage fbImage = FirebaseVisionImage.fromBitmap(img); - //Defines that will be used an on device text recognizer - FirebaseVisionTextRecognizer textRecognizer = FirebaseVision.getInstance().getOnDeviceTextRecognizer(); - textRecognizer.processImage(fbImage).addOnSuccessListener(new OnSuccessListener() { - @Override - public void onSuccess(FirebaseVisionText firebaseVisionText) { - //Saving of the retrieved text into shared preferences - storeText(firebaseVisionText.getText()); - //If there's some text the live data is updated so that can be updated the UI too - extractedText.setValue(firebaseVisionText.getText()); - } - }).addOnFailureListener(new OnFailureListener() { - @Override - public void onFailure(@NonNull Exception e) { - extractedText.setValue("No text retrieved"); - } - }); - - return extractedText.getValue(); - } - - - /** - * Saves the given text into the shared preferences so that it can be reused in the future - * @param text The text to save into the shared preferences - * @author Leonardo Rossi - */ - private void storeText(String text) { - SharedPreferences sharedPref = PreferenceManager.getDefaultSharedPreferences(context); - SharedPreferences.Editor editor = sharedPref.edit(); - editor.putString("text", text); - editor.commit(); - } -} diff --git a/app/src/main/java/unipd/se18/ocrcamera/Utils.java b/app/src/main/java/unipd/se18/ocrcamera/Utils.java new file mode 100644 index 00000000..d6c2a79c --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/Utils.java @@ -0,0 +1,86 @@ +package unipd.se18.ocrcamera; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; + +public class Utils { + /** + * @param filePath name of a jpeg file to convert to bitmap + * @return image converted to bitmap + */ + public static Bitmap loadBitmapFromFile(String filePath) { + Bitmap bitmap = BitmapFactory.decodeFile(filePath); + return bitmap; + } + + + /** + * @param filePath + * @return String with the text inside the file pointed by filePath, empty string if file doesn't exist + */ + public static String getTextFromFile(String filePath) { + + String text =""; + BufferedReader br = null; + try { + br = new BufferedReader(new FileReader(filePath)); + text = br.readLine(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + return text; + + } + + /** + * @param filePath path to file + * @return file extension if exists, null otherwise + */ + public static String getFileExtension(String filePath) { + int strLength = filePath.lastIndexOf("."); + if (strLength > 0) + return filePath.substring(strLength + 1).toLowerCase(); + return null; + } + + /** + * @param filePath path to file + * @return file name without extension if exists, null otherwise + */ + public static String getFilePrefix(String filePath) { + + int start = filePath.lastIndexOf("/") + 1; + int end = filePath.lastIndexOf("."); + if (end > start) + return filePath.substring(start, end); + return null; + } + + + /** + * @param json JSON object with containing the array given as param + * @param name array name in JSON object + * @return the array in the JSON object converted to String + */ + public static String[] getStringArrayFromJSON(JSONObject json, String name) throws JSONException { + JSONArray jsonArray = json.getJSONArray(name); + String[] array = new String[jsonArray.length()]; + for (int i = 0; i < jsonArray.length(); i++) { + array[i] = jsonArray.getString(i); + } + return array; + } + + +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/recognizer/MlKitRecognizer.java b/app/src/main/java/unipd/se18/ocrcamera/recognizer/MlKitRecognizer.java new file mode 100644 index 00000000..de1db97b --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/recognizer/MlKitRecognizer.java @@ -0,0 +1,202 @@ +package unipd.se18.ocrcamera.recognizer; +import android.graphics.Bitmap; +import android.support.annotation.NonNull; +import android.util.Log; + +import com.google.android.gms.tasks.OnFailureListener; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.android.gms.tasks.Task; +import com.google.firebase.ml.vision.FirebaseVision; +import com.google.firebase.ml.vision.common.FirebaseVisionImage; +import com.google.firebase.ml.vision.text.FirebaseVisionText; +import com.google.firebase.ml.vision.text.FirebaseVisionTextRecognizer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; + +/** + * Class that implements the common OCR interface for retrieving text from a Bitmap image. + * This class uses ml-kit provided by Firebase. + * More details at: {@link FirebaseVisionText}. + * @see + * ml-kit, recognize text + * @author Pietro Prandini (g2) + */ +class MlKitRecognizer implements OCR { + /** + * String used for the logs of this class. + */ + private final String TAG = "MlKitRecognizer -> "; + + /** + * The listener used to notify the result of the extraction + */ + private OCRListener textExtractionListener; + + /* + The next four int are used for recognizing the position of the FirebaseVisionText Objects. + They could be useful for sorting the blocks or for an automatic recognition + of the ingredients text block. + The indexes are a clockwise order from the top-left corner. + More details at: + {@link FirebaseVisionText.TextBlock#getCornerPoints()}, + {@link FirebaseVisionText.Line#getCornerPoints()} + or {@link FirebaseVisionText.Element#getCornerPoints()}. + */ + private final int TOP_LEFT = 0; + private final int TOP_RIGHT = 1; + private final int BOTTOM_LEFT = 2; + private final int BOTTOM_RIGHT = 3; + + /** + * Constructor of this recognizer + * @param textExtractionListener The listener used to notify the result of the extraction + * @author Pietro Prandini (g2) + */ + MlKitRecognizer(OCRListener textExtractionListener) { + this.textExtractionListener = textExtractionListener; + } + + /* + The next method is required by the OCRInterface that avoid a single point of failure. + */ + + /** + * Extracts a text from a given image. + * More details at: {@link FirebaseVisionText}. + * @param img The image in a Bitmap format + * @author Pietro Prandini (g2) + */ + public void getTextFromImg(Bitmap img) { + // String used for the logs of this method + final String methodTag = "getTextFromImg -> "; + Log.d(TAG, methodTag + "launched"); + + // Extracts the text from the pic + extractFireBaseVisionText(img); + } + + /** + * Extracts a FirebaseVisionText from a given image. + * More details at: {@link FirebaseVisionText}, {@link CountDownLatch}, + * {@link Task#addOnSuccessListener(OnSuccessListener)}, {@link OnSuccessListener}. + * @param img The image in a Bitmap format + * @author Pietro Prandini (g2) + */ + private void extractFireBaseVisionText(Bitmap img) { + // String used for the logs of this method + final String methodTag = "extractFireBaseVisionText -> "; + Log.d(TAG, methodTag + "launched"); + + // Starts the time counter - useful for tests + final long beforeWaiting = java.lang.System.currentTimeMillis(); + + // Settings the image to analyze + FirebaseVisionImage firebaseVisionImage = FirebaseVisionImage.fromBitmap(img); + + // Settings the "on device" analyzing method + FirebaseVisionTextRecognizer textRecognizer = + FirebaseVision.getInstance().getOnDeviceTextRecognizer(); + + // Settings the extraction task + textRecognizer.processImage(firebaseVisionImage).addOnSuccessListener( + new OnSuccessListener() { + @Override + public void onSuccess(FirebaseVisionText firebaseVisionText) { + Log.v(TAG, methodTag + "onSuccess ->\n" + + "----- RECOGNIZED TEXT -----" + + "\n" + firebaseVisionText.getText() + "\n" + + "----- END OF THE RECOGNIZED TEXT -----"); + + // Ends the time counter - useful for tests + long afterWaiting = java.lang.System.currentTimeMillis(); + Log.i(TAG, methodTag + "text extracted in " + + (afterWaiting - beforeWaiting) + " ms"); + + // Notify to the listener the result of the task + textExtractionListener.onTextRecognized(extractString(firebaseVisionText)); + } + }).addOnFailureListener(new OnFailureListener() { + @Override + public void onFailure(@NonNull Exception e) { + Log.e(TAG, "Error: " + e); + + // Notify to the listener the result of the task + textExtractionListener.onTextRecognizedError(OCRListener.FAILURE); + } + }); + } + + /** + * Produces a String from a FirebaseVisionText + * More details at: {@link FirebaseVisionText.TextBlock#getText()}. + * @param firebaseVisionTextExtracted The result of the FirebaseVisionText extraction + * @return String extracted by the FirebaseVisionText result + * @author Pietro Prandini (g2) + */ + private String extractString(FirebaseVisionText firebaseVisionTextExtracted) { + // Orders the blocks (requested by the issue #18 but not so useful at the moment) + ArrayList textBlocks = + sortBlocks(firebaseVisionTextExtracted); + + // Prepares the String with the text of the blocks identified by the Firebase process + StringBuilder text = new StringBuilder(); + for(FirebaseVisionText.TextBlock block: textBlocks) { + text.append(block.getText()).append("\n"); + } + String extractedText = text.toString(); + + // Returns the String + return extractedText; + } + + /* + Sorting methods are not so useful at the moment. + There is an issue (#18) that request this method for a possible future use. + */ + + /** + * Sorts the blocks recognized in an ArrayList + * More details at: {@link FirebaseVisionText.TextBlock#getTextBlocks()}. + * @param OCRResult FirebaseVisionText object produced by an OCR recognition + * @return An ArrayList of FirebaseVisionText sorted + * @author Pietro Prandini (g2) + */ + private ArrayList sortBlocks(FirebaseVisionText OCRResult) { + ArrayList OCRBlocks = + new ArrayList<>(OCRResult.getTextBlocks()); + // Sorts the ArrayList of the blocks from top to bottom + OCRBlocks = sortBlocksY(OCRBlocks); + return OCRBlocks; + } + + /** + * Sorts the blocks recognized from top to bottom in an ArrayList + * More details at: {@link FirebaseVisionText.TextBlock#getCornerPoints()}, {@link Comparator}, + * {@link Collections#sort(List, Comparator)}. + * @param OCRBlocks ArrayList of FirebaseVisionText.TextBlock recognized by the OCR processing + * @return An ArrayList of FirebaseVisionText sorted from top to bottom + * @author Pietro Prandini (g2) + */ + private ArrayList + sortBlocksY(ArrayList OCRBlocks) { + // Comparator for ordering the blocks by the y axis + Comparator mYComparator = + new Comparator() { + @Override + public int compare(FirebaseVisionText.TextBlock o1, + FirebaseVisionText.TextBlock o2) { + int o1TopLeftY = Objects.requireNonNull(o1.getCornerPoints())[TOP_LEFT].y; + int o2TopLeftY = Objects.requireNonNull(o2.getCornerPoints())[TOP_LEFT].y; + return Integer.compare(o1TopLeftY,o2TopLeftY); + } + }; + // Sorts the blocks + Collections.sort(OCRBlocks, mYComparator); + return OCRBlocks; + } +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/recognizer/OCR.java b/app/src/main/java/unipd/se18/ocrcamera/recognizer/OCR.java new file mode 100644 index 00000000..fbb72ede --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/recognizer/OCR.java @@ -0,0 +1,17 @@ +package unipd.se18.ocrcamera.recognizer; + +import android.graphics.Bitmap; + +/** + * Interface useful to avoid the single point of failure about the OCR recognizing text + * @author Commonly decided by all the groups + */ +public interface OCR { + /** + * Launches the text recognizing process from a given image. + * See OCRListener.java of this package for retrieving the output of this process. + * @param img The image in the Bitmap format + * @author Commonly decided by all the groups, modified by a suggestion from the doctor Li Daohong + */ + void getTextFromImg(Bitmap img); +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/recognizer/OCRListener.java b/app/src/main/java/unipd/se18/ocrcamera/recognizer/OCRListener.java new file mode 100644 index 00000000..76df72d0 --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/recognizer/OCRListener.java @@ -0,0 +1,24 @@ +package unipd.se18.ocrcamera.recognizer; + +/** + * Listener useful to control the output of the OCR processing + * @author Pietro Prandini (g2) - suggested by the doctor Li Daohong + */ +public interface OCRListener { + /** + * Code of extraction failure + */ + int FAILURE = 0; + + /** + * Method called when an extraction is successfully completed. + * @param text The String of the text recognized (empty String if nothing is recognized) + */ + void onTextRecognized(String text); + + /** + * Method called when an extraction is failed. + * @param code The code of the extraction error + */ + void onTextRecognizedError(int code); +} diff --git a/app/src/main/java/unipd/se18/ocrcamera/recognizer/TextRecognizer.java b/app/src/main/java/unipd/se18/ocrcamera/recognizer/TextRecognizer.java new file mode 100644 index 00000000..1f2e857c --- /dev/null +++ b/app/src/main/java/unipd/se18/ocrcamera/recognizer/TextRecognizer.java @@ -0,0 +1,27 @@ +package unipd.se18.ocrcamera.recognizer; + +/** + * Class useful to set a type of recognizing. + * It's avoid the single point of failure relative to this process. + * @author Pietro Prandini (g2) - suggested by the doctor Li Daohong + */ +public class TextRecognizer { + /** + * Ids of the ocr recognizers + */ + public enum Recognizer { mlKit } + + /** + * Provides an OCR recognizer + * @param type The id of the recognizing type requested + * @param textRecognizerListener The listener used to notify the result of the extraction + * @return The OCR object corresponding to the id of the recognizing type requested + * @author Pietro Prandini (g2) + */ + public static OCR getTextRecognizer(Recognizer type, OCRListener textRecognizerListener) { + switch (type) { + case mlKit: return new MlKitRecognizer(textRecognizerListener); + default: return new MlKitRecognizer(textRecognizerListener); + } + } +} diff --git a/app/src/main/res/drawable-v24/ic_photo_camera.xml b/app/src/main/res/drawable-v24/ic_photo_camera.xml new file mode 100644 index 00000000..b627a0bb --- /dev/null +++ b/app/src/main/res/drawable-v24/ic_photo_camera.xml @@ -0,0 +1,6 @@ + + + + + diff --git a/app/src/main/res/drawable/ic_photo_camera_24px.xml b/app/src/main/res/drawable/ic_photo_camera_24px.xml new file mode 100644 index 00000000..84aed6db --- /dev/null +++ b/app/src/main/res/drawable/ic_photo_camera_24px.xml @@ -0,0 +1,12 @@ + + + + diff --git a/app/src/main/res/layout-land/activity_camera.xml b/app/src/main/res/layout-land/activity_camera.xml new file mode 100644 index 00000000..3cfc73b4 --- /dev/null +++ b/app/src/main/res/layout-land/activity_camera.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/layout-land/activity_result.xml b/app/src/main/res/layout-land/activity_result.xml new file mode 100644 index 00000000..d89ff483 --- /dev/null +++ b/app/src/main/res/layout-land/activity_result.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/layout/activity_camera.xml b/app/src/main/res/layout/activity_camera.xml index caccb7fb..61e61a49 100644 --- a/app/src/main/res/layout/activity_camera.xml +++ b/app/src/main/res/layout/activity_camera.xml @@ -6,34 +6,41 @@ android:layout_height="match_parent" tools:context=".CameraActivity"> + - + + -