Skip to content

Commit

Permalink
* aspect detection and options tuning
Browse files Browse the repository at this point in the history
  • Loading branch information
@allgood
allgood committed Oct 7, 2022
1 parent 9a35d3e commit 8deea61
Show file tree
Hide file tree
Showing 10 changed files with 367 additions and 78 deletions.
115 changes: 93 additions & 22 deletions app/src/main/java/com/todobom/opennotescanner/ImageProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import android.graphics.Paint;
import android.graphics.Path;
import android.graphics.drawable.shapes.PathShape;
import android.hardware.SensorManager;
import android.os.Handler;
import android.os.Looper;
import android.os.Message;
Expand All @@ -28,7 +29,6 @@
import com.todobom.opennotescanner.helpers.Utils;
import com.todobom.opennotescanner.views.HUDCanvasView;

import org.jetbrains.annotations.NonNls;
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
Expand Down Expand Up @@ -65,12 +65,26 @@ public class ImageProcessor extends Handler {
private Size mPreviewSize;
private Point[] mPreviewPoints;

private double mDocumentAspectRatio;

public ImageProcessor(Looper looper, OpenNoteScannerActivity mainActivity) {
super(looper);
mMainActivity = mainActivity;

SharedPreferences sharedPref = PreferenceManager.getDefaultSharedPreferences(mainActivity);
mBugRotate = sharedPref.getBoolean("bug_rotate",false);

String docPageFormat = sharedPref.getString("document_page_format", "0");
mDocumentAspectRatio = 0;
if (docPageFormat.equals("0.0001")) {
Float customPageWidth = Float.parseFloat(sharedPref.getString("custom_pageformat_width" , "0" ));
Float customPageHeight = Float.parseFloat(sharedPref.getString("custom_pageformat_height" , "0" ));
if (customPageWidth > 0 && customPageHeight > 0) {
mDocumentAspectRatio = customPageHeight / customPageWidth;
}
} else {
mDocumentAspectRatio = Float.parseFloat(docPageFormat);
}
}

public void handleMessage ( Message msg ) {
Expand Down Expand Up @@ -216,6 +230,7 @@ private boolean detectPreviewDocument(Mat inputRgba) {

mPreviewPoints = null;
mPreviewSize = inputRgba.size();
drawDocumentArea(mPreviewSize);

if (quad != null) {

Expand Down Expand Up @@ -250,14 +265,80 @@ private boolean detectPreviewDocument(Mat inputRgba) {

}

private void drawDocumentArea(Size stdSize) {
HUDCanvasView hud = mMainActivity.getHUD();

if (mDocumentAspectRatio == 0) {
hud.setDocumentBoxShape(null, null, null);
return;
}

/* SAVING TO USE ON ANOTHER PLACE * /
final float[] rotationMatrix = new float[9];
SensorManager.getRotationMatrix(rotationMatrix, null,
mMainActivity.accelerometerReading, mMainActivity.magnetometerReading);
// Express the updated rotation matrix as three orientation angles.
final float[] orientationAngles = new float[3];
SensorManager.getOrientation(rotationMatrix, orientationAngles);
// Pitch will vary from -1,57 (full vertical) to 0 (full horizontal) to +1,57 (upside down)
float pitch = 0; // orientationAngles[1]; // for now, ignore inclination
// force 0 to 45 degrees
if (pitch < -0.78) {
pitch = (float) -0.78;
} else if (pitch > 0) {
pitch = 0;
}
/* */

// ATTENTION: axis are swapped
float previewWidth = (float) stdSize.height;
float previewHeight = (float) stdSize.width;

Path path = new Path();

path.moveTo( 0 ,0 );
path.lineTo( previewWidth , 0 );
path.lineTo( previewWidth, previewHeight );
path.lineTo( 0, previewHeight );
path.close();

float previewAspectRatio = previewHeight/previewWidth;

int[] documentArea = Utils.getDocumentArea((int) previewHeight, (int) previewWidth, mMainActivity);

path.moveTo( documentArea[1] , documentArea[0] );
path.lineTo( documentArea[1] , documentArea[2] );
path.lineTo( documentArea[3] , documentArea[2] );
path.lineTo( documentArea[3] , documentArea[0] );
path.close();

path.setFillType(Path.FillType.EVEN_ODD);

PathShape newBox = new PathShape(path , previewWidth , previewHeight);

Paint paint = new Paint();
paint.setColor(Color.argb(64, 255, 255, 255));

Paint border = new Paint();
border.setColor(Color.argb( 32, 255, 255, 255));
border.setStrokeWidth(5);

hud.setDocumentBoxShape(newBox, paint, border);
mMainActivity.invalidateHUD();
}

private void drawDocumentBox(Point[] points, Size stdSize) {

Path path = new Path();

HUDCanvasView hud = mMainActivity.getHUD();

// ATTENTION: axis are swapped

float previewWidth = (float) stdSize.height;
float previewHeight = (float) stdSize.width;

Expand All @@ -277,10 +358,8 @@ private void drawDocumentBox(Point[] points, Size stdSize) {
border.setStrokeWidth(5);

hud.clear();
hud.addShape(newBox, paint, border);
hud.setDetectedShape(newBox, paint, border);
mMainActivity.invalidateHUD();


}

private Quadrilateral getQuadrilateral( ArrayList<MatOfPoint> contours , Size srcSize ) {
Expand All @@ -302,7 +381,7 @@ private Quadrilateral getQuadrilateral( ArrayList<MatOfPoint> contours , Size sr
if (points.length == 4) {
Point[] foundPoints = sortPoints(points);

if (insideArea(foundPoints, size)) {
if (insideHotArea(foundPoints, size)) {
return new Quadrilateral( c , foundPoints );
}
}
Expand Down Expand Up @@ -336,23 +415,15 @@ private Point[] sortPoints( Point[] src ) {
return result;
}

private boolean insideArea(Point[] rp, Size size) {
private boolean insideHotArea(Point[] rp, Size size) {

int width = Double.valueOf(size.width).intValue();
int height = Double.valueOf(size.height).intValue();
int baseMeasure = height/4;

int bottomPos = height-baseMeasure;
int topPos = baseMeasure;
int leftPos = width/2-baseMeasure;
int rightPos = width/2+baseMeasure;
int[] hotArea = Utils.getHotArea((int) size.width, (int) size.height, this.mMainActivity);

return (
rp[0].x <= leftPos && rp[0].y <= topPos
&& rp[1].x >= rightPos && rp[1].y <= topPos
&& rp[2].x >= rightPos && rp[2].y >= bottomPos
&& rp[3].x <= leftPos && rp[3].y >= bottomPos

rp[0].x <= hotArea[0] && rp[0].y <= hotArea[1]
&& rp[1].x >= hotArea[2] && rp[1].y <= hotArea[1]
&& rp[2].x >= hotArea[2] && rp[2].y >= hotArea[3]
&& rp[3].x <= hotArea[0] && rp[3].y >= hotArea[3]
);
}

Expand All @@ -365,7 +436,7 @@ private void enhanceDocument( Mat src ) {
Mat copy = new Mat(src.size(), CvType.CV_8UC3);
src.copyTo(copy);

Imgproc.adaptiveThreshold(mask,mask,255,Imgproc.ADAPTIVE_THRESH_MEAN_C,Imgproc.THRESH_BINARY_INV,15,15);
Imgproc.adaptiveThreshold(mask,mask,255,Imgproc.ADAPTIVE_THRESH_MEAN_C,Imgproc.THRESH_BINARY_INV,225,15);

src.setTo(new Scalar(255,255,255));
copy.copyTo(src,mask);
Expand All @@ -378,7 +449,7 @@ private void enhanceDocument( Mat src ) {
} else if (!colorMode) {
Imgproc.cvtColor(src,src,Imgproc.COLOR_RGBA2GRAY);
if (filterMode) {
Imgproc.adaptiveThreshold(src, src, 255, Imgproc.ADAPTIVE_THRESH_MEAN_C, Imgproc.THRESH_BINARY, 15, 15);
Imgproc.adaptiveThreshold(src, src, 255, Imgproc.ADAPTIVE_THRESH_MEAN_C, Imgproc.THRESH_BINARY, 225, 15);
}
}
}
Expand Down
90 changes: 78 additions & 12 deletions app/src/main/java/com/todobom/opennotescanner/OpenNoteScannerActivity.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,15 @@ package com.todobom.opennotescanner
import android.Manifest
import android.annotation.SuppressLint
import android.app.AlertDialog
import android.content.Context
import android.content.DialogInterface
import android.content.Intent
import android.content.SharedPreferences
import android.content.pm.PackageManager
import android.content.res.ColorStateList
import android.graphics.Point
import android.graphics.Rect
import android.hardware.Camera
import android.hardware.*
import android.hardware.Camera.*
import android.media.AudioManager
import android.media.MediaPlayer
Expand Down Expand Up @@ -53,7 +54,8 @@ import java.util.*
* status bar and navigation/system bar) with user interaction.
*/
class OpenNoteScannerActivity : AppCompatActivity(), NavigationView.OnNavigationItemSelectedListener,
SurfaceHolder.Callback, PictureCallback, PreviewCallback, SetTopicDialogListener {
SurfaceHolder.Callback, PictureCallback, PreviewCallback, SetTopicDialogListener, SensorEventListener {
var mDocumentAspectRatio: Double = 0.0
private val mHideHandler = Handler()
private lateinit var mContentView: View
private val mHidePart2Runnable = Runnable { // Delayed removal of status and navigation bar
Expand Down Expand Up @@ -96,6 +98,10 @@ class OpenNoteScannerActivity : AppCompatActivity(), NavigationView.OnNavigation
private var mat: Mat? = null
private lateinit var tracker: Tracker

private lateinit var sensorManager: SensorManager
lateinit var accelerometerReading: FloatArray
lateinit var magnetometerReading: FloatArray

fun setImageProcessorBusy(imageProcessorBusy: Boolean) {
this.imageProcessorBusy = imageProcessorBusy
}
Expand All @@ -108,6 +114,10 @@ class OpenNoteScannerActivity : AppCompatActivity(), NavigationView.OnNavigation
private var attemptToFocus = false
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)

accelerometerReading = FloatArray(3);
magnetometerReading = FloatArray(3);

mThis = this
mSharedPref = PreferenceManager.getDefaultSharedPreferences(this)
if (mSharedPref.getBoolean("isFirstRun", true) && !mSharedPref.getBoolean("usage_stats", false)) {
Expand All @@ -122,6 +132,7 @@ class OpenNoteScannerActivity : AppCompatActivity(), NavigationView.OnNavigation
hUD = findViewById(R.id.hud)
mWaitSpinner = findViewById(R.id.wait_spinner)

sensorManager = getSystemService(Context.SENSOR_SERVICE) as SensorManager

// Set up the user interaction to manually show or hide the system UI.
mContentView.setOnClickListener { toggle() }
Expand Down Expand Up @@ -232,6 +243,7 @@ class OpenNoteScannerActivity : AppCompatActivity(), NavigationView.OnNavigation
}
}

@SuppressLint("MissingSuperCall")
override fun onRequestPermissionsResult(requestCode: Int,
permissions: Array<String>, grantResults: IntArray) {
when (requestCode) {
Expand Down Expand Up @@ -317,6 +329,24 @@ class OpenNoteScannerActivity : AppCompatActivity(), NavigationView.OnNavigation

public override fun onResume() {
super.onResume()

sensorManager.getDefaultSensor(Sensor.TYPE_ACCELEROMETER)?.also { accelerometer ->
sensorManager.registerListener(
this,
accelerometer,
SensorManager.SENSOR_DELAY_NORMAL,
SensorManager.SENSOR_DELAY_UI
)
}
sensorManager.getDefaultSensor(Sensor.TYPE_MAGNETIC_FIELD)?.also { magneticField ->
sensorManager.registerListener(
this,
magneticField,
SensorManager.SENSOR_DELAY_NORMAL,
SensorManager.SENSOR_DELAY_UI
)
}

window.decorView.systemUiVisibility = (View.SYSTEM_UI_FLAG_FULLSCREEN
or View.SYSTEM_UI_FLAG_LAYOUT_STABLE
or View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN
Expand Down Expand Up @@ -357,6 +387,9 @@ class OpenNoteScannerActivity : AppCompatActivity(), NavigationView.OnNavigation
public override fun onDestroy() {
super.onDestroy()
// FIXME: check disableView()

// Don't receive any more updates from either sensor.
sensorManager.unregisterListener(this)
}

val resolutionList: List<Camera.Size>
Expand Down Expand Up @@ -490,27 +523,46 @@ class OpenNoteScannerActivity : AppCompatActivity(), NavigationView.OnNavigation
surfaceView.layoutParams = surfaceParams
ud.layoutParams.height = previewHeight
}
val hotAreaWidth = displayWidth / 4
val hotAreaHeight = previewHeight / 2 - hotAreaWidth
val hotAreaSpaceWidth: Int
val hotAreaSpaceHeight: Int

val sharedPref = PreferenceManager.getDefaultSharedPreferences(this)
val docPageFormat: String? = sharedPref.getString("document_page_format", "0")
this.mDocumentAspectRatio = 0.0
if (docPageFormat == "0.0001") {
val customPageWidth: Float = sharedPref.getString("custom_pageformat_width", "0")!!.toFloat()
val customPageHeight: Float = sharedPref.getString("custom_pageformat_height", "0")!!.toFloat()
if (customPageWidth > 0 && customPageHeight > 0) {
this.mDocumentAspectRatio = (customPageHeight / customPageWidth).toDouble()
}
} else {
this.mDocumentAspectRatio = docPageFormat!!.toFloat().toDouble()
}

var hotArea = Utils.getHotArea(pSize.width, pSize.height, this)

hotAreaSpaceWidth = hotArea!![1]
hotAreaSpaceHeight = hotArea!![0]

val angleNorthWest = findViewById<ImageView>(R.id.nw_angle)
val paramsNW = angleNorthWest.layoutParams as RelativeLayout.LayoutParams
paramsNW.leftMargin = hotAreaWidth - paramsNW.width
paramsNW.topMargin = hotAreaHeight - paramsNW.height
paramsNW.leftMargin = hotAreaSpaceWidth - paramsNW.width
paramsNW.topMargin = hotAreaSpaceHeight - paramsNW.height
angleNorthWest.layoutParams = paramsNW
val angleNorthEast = findViewById<ImageView>(R.id.ne_angle)
val paramsNE = angleNorthEast.layoutParams as RelativeLayout.LayoutParams
paramsNE.leftMargin = displayWidth - hotAreaWidth
paramsNE.topMargin = hotAreaHeight - paramsNE.height
paramsNE.leftMargin = displayWidth - hotAreaSpaceWidth
paramsNE.topMargin = hotAreaSpaceHeight - paramsNE.height
angleNorthEast.layoutParams = paramsNE
val angleSouthEast = findViewById<ImageView>(R.id.se_angle)
val paramsSE = angleSouthEast.layoutParams as RelativeLayout.LayoutParams
paramsSE.leftMargin = displayWidth - hotAreaWidth
paramsSE.topMargin = previewHeight - hotAreaHeight
paramsSE.leftMargin = displayWidth - hotAreaSpaceWidth
paramsSE.topMargin = previewHeight - hotAreaSpaceHeight
angleSouthEast.layoutParams = paramsSE
val angleSouthWest = findViewById<ImageView>(R.id.sw_angle)
val paramsSW = angleSouthWest.layoutParams as RelativeLayout.LayoutParams
paramsSW.leftMargin = hotAreaWidth - paramsSW.width
paramsSW.topMargin = previewHeight - hotAreaHeight
paramsSW.leftMargin = hotAreaSpaceWidth - paramsSW.width
paramsSW.topMargin = previewHeight - hotAreaSpaceHeight
angleSouthWest.layoutParams = paramsSW
val maxRes = getMaxPictureResolution(previewRatio)
if (maxRes != null) {
Expand Down Expand Up @@ -791,4 +843,18 @@ class OpenNoteScannerActivity : AppCompatActivity(), NavigationView.OnNavigation
private const val RESUME_PERMISSIONS_REQUEST_CAMERA = 11
private const val TAG = "OpenNoteScannerActivity"
}

// Get readings from accelerometer and magnetometer. To simplify calculations,
// consider storing these readings as unit vectors.
override fun onSensorChanged(event: SensorEvent) {
if (event.sensor.type == Sensor.TYPE_ACCELEROMETER) {
System.arraycopy(event.values, 0, accelerometerReading, 0, accelerometerReading.size)
} else if (event.sensor.type == Sensor.TYPE_MAGNETIC_FIELD) {
System.arraycopy(event.values, 0, magnetometerReading, 0, magnetometerReading.size)
}
}

override fun onAccuracyChanged(p0: Sensor?, p1: Int) {
// TODO("Not yet implemented")
}
}
Loading

0 comments on commit 8deea61

Please sign in to comment.