/* * GrabScreen * * Use JavaRobot to Grab the screen * * * Inspired by: * https://alvinalexander.com/java/java-robot-class-example-mouse-keystroke * * ideas:- * * Grab screen and watch the screen for a Change. Exit on screen change. * Saves screen capture as .png * * usage: * javac GrabScreen * java GrabScreen filename x y w d * java GrabScreen im1 5 30 40 40 * image contains a clip off the screen We need to processit to classify it. We need to do some image arithmatics. Tests: Is the image blank? For each pixel, are they equal? Is the image complex? For each pixel, work out the mean, standard deviation Is the image complex? Each Pixel is RGB, Is the image dark? For each pixel sum the pixels. What colour? for each PIXEL work out which is max R,G or B and report R G B = " | diffs: " + (r-g) + ", " + (g-b) + ", " + (b-r) * * * * * * * * * This program can grab some of the screen, but does not know what it is. The window is focused, but we do not khow where it is. Look for the edges. Grab a thin letterbox and from the center look for the edges. Grab a thin Portrate and from the center look for the edges. Using these coordinates grab the image of interest. define the areas to watch. wait till an area changes and return. capture before and after screen shots Could have a */ /* ==================================== */ import java.io.*; import java.awt.AWTException; import java.awt.Robot; import java.awt.event.InputEvent; import java.awt.event.KeyEvent; /* ==================================== */ import java.awt.Color; import java.awt.Rectangle; import java.awt.image.BufferedImage; import javax.swing.JFrame; import javax.swing.JMenu; import javax.swing.JMenuBar; import javax.swing.JMenuItem; import javax.swing.SwingUtilities; import javax.swing.UIManager; /* ==================================== */ import javax.imageio.*; import java.awt.MouseInfo; import java.awt.Point; import java.awt.PointerInfo; import java.awt.event.MouseListener; import java.awt.event.MouseEvent; import javax.swing.*; /* ==================================== */ //https://docs.oracle.com/javase/7/docs/api/java/awt/GraphicsDevice.html import java.awt.GraphicsDevice; /* ==================================== */ public class GrabScreen { Robot robot = new Robot(); class icon { /* Set up list of images */ Rectangle rect ; BufferedImage firstImage; BufferedImage image; }; static int lx; static int ly; static int ic = 0 ; static boolean d = true; static icon[] icons = new icon[ 100 ] ; // byte dispMemRead[] = new byte[16]; /* tracks key presses - */ public static void main(String[] args) throws AWTException { System.out.println( "usage: java GrabsScreen img x y w h - waits for image to change" ); System.out.println( "usage: java GrabsScreen - mouse co-ordinates" ); /* if no parameters show mouse co-ords */ if ( args.length > 4 ) { System.out.println("starting...."); System.out.println("args[0]: " + args[0] ); System.out.println("args[1]: " + args[1] ); System.out.println("args[2]: " + args[2] ); System.out.println("args[3]: " + args[3] ); System.out.println("args[4]: " + args[4] ); new GrabScreen( args[0], Integer.parseInt(args[1]), Integer.parseInt(args[2]), Integer.parseInt(args[3]), Integer.parseInt(args[4]) ); } else { mouseCapture( ); } } public GrabScreen( String fileName , int rx, int ry, int rw, int rh ) throws AWTException { int count =0; int i ; robot.setAutoDelay(40); robot.setAutoWaitForIdle(true); Rectangle[] rects = new Rectangle[ 10 ] ; BufferedImage[] firstImages = new BufferedImage[ 10 ]; BufferedImage[] images = new BufferedImage[ 10 ]; int[] matchRanks = new int[ 10 ]; /* Set up list of images */ Rectangle rect = new Rectangle( rx , ry , rw , rh ); BufferedImage firstImage = robot.createScreenCapture( rect ); BufferedImage image = robot.createScreenCapture( rect ); /* save image */ saveImage( fileName , image ); /* options rectangle */ count = 1; rects[ count++ ] = new Rectangle( 564,327,34,24 ); //option rects[ count++ ] = new Rectangle( 1314,326,34,27 ); // calls rects[ count++ ] = new Rectangle( 668,415,569,288 ); // rects[ count++ ] = new Rectangle( 946,332,23,18 ); // green rects[ count++ ] = new Rectangle( 969,350,10,77 ); rects[ count++ ] = new Rectangle( 560,297,28,14 ); // corner rects[ count++ ] = new Rectangle( 588,311,190,169 ); //count--; System.out.println( "count:" + count ); for( i=1 ; i < count ; i++){ firstImages[i] = robot.createScreenCapture( rects[i] ); saveImage( (fileName+"_"+i) , firstImages[i] ); } /* save image */ //saveImage( fileName+"_1-1" , firstImages[1] ); //saveImage( fileName+"_2-1" , firstImages[2] ); /* rect = new Rectangle( 0 , 500 , 1000 , 10 ); BufferedImage letterbox = robot.createScreenCapture( rect ); findEdge( letterbox ); System.exit(0); */ int matchRank = testImage( image ); System.out.println( "M:" + matchRank ); //reportImage( image ); System.out.println( "count:" + count ); while( matchRank > 0 ){ image = robot.createScreenCapture( rect ); matchRank = testImage( image, firstImage ); System.out.println( "M:" + matchRank ); for( i = 1; i < count ; i++){ images[ i ] = robot.createScreenCapture( rects[ i ] ); matchRanks[ i ] = testImage( images[ i ], firstImages[ i ] ); System.out.print( matchRanks[i] +" " ); } robot.delay(500); } // saveImage( fileName+("_2") , image ); // saveImage( fileName+("_1-2") , images[1] ); // saveImage( fileName+("_2-2") , images[2] ); for( i=1 ; i < count ; i++){ saveImage( (fileName+"_"+i+"_end") , images[i] ); } System.exit(0); } private void reportImage( BufferedImage image ) { /* * look at image find middle and look at it. */ int y = image.getHeight() / 2; Color lastColor = new Color( image.getRGB( 0, y )); Color c = new Color( image.getRGB( 0 ,y )); for (int x = 0; x < image.getWidth(); x++) { lastColor = c; c = new Color( image.getRGB( x, y ) ); int r,g,b,sum; r = c.getRed(); g = c.getGreen(); b = c.getBlue(); sum = r+g+b; /* r-g,g-b,b-r */ // System.out.println( x + "," + y + "," + c.equals( lastColor ) ); // System.out.println( c +" | "+c.toString()+" |hash "+c.hashCode() ); System.out.println( x + "," + y + "," + c +" | hash: "+c.hashCode() + " | diffs: " + (r-g) + ", " + (g-b) + ", " + (b-r) ); if (c.equals(Color.BLUE)) { //https://docs.oracle.com/javase/7/docs/api/java/awt/Color.html // dispose(); // throw new RuntimeException("JMenu Disabled" + " State not Valid."); // System.out.println("JMenu Disabled...."); } } } private int testImage( BufferedImage image ) { /* * look at image find middle and looks at it. */ int y = image.getHeight() / 2; Color lastColor = new Color( image.getRGB( 0, y )); Color c = new Color( image.getRGB( 0 ,y )); int sumEquals = 0; for (int x = 0; x < image.getWidth(); x++) { lastColor = c; c = new Color( image.getRGB( x, y ) ); int r,g,b,sum; r = c.getRed(); g = c.getGreen(); b = c.getBlue(); sum = r+g+b; /* r-g,g-b,b-r */ if ( c.equals( lastColor ) ) { sumEquals += 1; } else { sumEquals += -1; }; System.out.println( x + "," + y + "," + c.equals( lastColor ) ); /* if (c.equals(Color.BLUE)) { //https://docs.oracle.com/javase/7/docs/api/java/awt/Color.html // dispose(); // throw new RuntimeException("JMenu Disabled" + " State not Valid."); // System.out.println("JMenu Disabled...."); } */ } return sumEquals; } private int testImage( BufferedImage image, BufferedImage firstImage ) { /* * look at images find middle and compare to firstImage. */ int y = image.getHeight() / 2; Color firstC = new Color( image.getRGB( 0, y )); Color c = new Color( image.getRGB( 0 ,y )); int sumEquals = 0; for (int x = 0; x < image.getWidth(); x++) { firstC = new Color( firstImage.getRGB( x, y ) ); c = new Color( image.getRGB( x, y ) ); if ( c.equals( firstC ) ) { sumEquals += 1; } else { sumEquals += -1; }; //System.out.println( x + "," + y + "," + c.equals( firstC ) ); } return sumEquals; } /* * * find edge. start in 'middle' and work right till significant change found. * */ private int findEdge( BufferedImage image ) { /* * look at image find middle and looks at it. */ int y = image.getHeight()/2; int mx = image.getWidth()/2; mx = 0; int deltaX = image.getWidth()/100; if( deltaX < 1 ) { deltaX = 1; } System.out.println( "mx:" + mx + "," + y ); Color lastColor = new Color( image.getRGB( mx, y )); Color c = new Color( image.getRGB( mx ,y )); System.out.println( "===:" + mx + "," + y + "," + c.equals( lastColor )+"|"+lastColor + "!=" + c ); int edgeX = 0; int x = mx; while( edgeX == 0 ){ lastColor = c; //System.out.println( "+++:" + x + "," + y + "," + c.equals( lastColor )+"|"+lastColor + "!=" + c ); if ( x < image.getWidth() ){ c =new Color( image.getRGB( x, y ) ); } if ( !c.equals( lastColor ) ) { edgeX = x; System.out.println( x + "," + y + "," + c.equals( lastColor )+"|"+lastColor + "!=" + c ); } x += deltaX; if ( x > image.getWidth() ){ edgeX = -1; } } // relative to image edge not screen edge. System.out.println( edgeX ); return edgeX; } private void saveImage( String fileName , BufferedImage image ) { System.out.println( "filename:" + fileName ); try { // retrieve image // File outputfile = new File("saved.png"); File outputfile = new File( fileName + ".png"); ImageIO.write(image, "png", outputfile); } catch (IOException e) { System.out.println("image save failed..." ); } } private void leftClick() { robot.mousePress(InputEvent.BUTTON1_MASK); robot.delay(200); robot.mouseRelease(InputEvent.BUTTON1_MASK); robot.delay(200); } private void type(int i) { robot.delay(10); robot.keyPress(i); robot.keyRelease(i); } private void type(String s) { byte[] bytes = s.getBytes(); for (byte b : bytes) { int code = b; // keycode only handles [A-Z] (which is ASCII decimal [65-90]) if (code > 96 && code < 123) code = code - 32; // robot.delay(40); robot.delay(10); robot.keyPress(code); robot.keyRelease(code); } } private void script() { /* robot.delay(4000); robot.mouseMove(40, 130); robot.delay(500); // select App to bring to the front. leftClick(); robot.delay(500); type("\n\n\n\n"); robot.mouseMove(40, 130); robot.delay(500); leftClick(); robot.delay(500); leftClick(); robot.delay(500); type("Hello, world"); robot.mouseMove(40, 160); robot.delay(500); leftClick(); robot.delay(500); leftClick(); robot.delay(500); type("This is a test of the Java Robot class"); robot.delay(50); type(KeyEvent.VK_DOWN); robot.delay(50); type("Four score and seven years ago, our fathers ..."); robot.delay(1000); */ } /************************************************ * * This routine reports the cursor position. * * If the mouse pauses, the co-ordinates are output. * * x,y, rel_x, rel_y, @ , base_x,base_y * * It is possible to set a base point, by moving the cursor to 0,0 * then to the new base point * By moving the cursot to the top left or 0,0, * waiting and then to the new point. * * Output the last x,y,w,h * & *************************************************/ public static int mouseCapture( ) { /* base set:, 625, 137 delta:625,137 :0,0 @625,137 delta:627,166 :2,29 @625,137 delta:666,194 :41,57 @625,137 delta:1005,165 :380,28 @625,137 delta:1033,189 :408,52 @625,137 delta:1378,165 :753,28 @625,137 delta:1415,192 :790,55 @625,137 delta:844,319 :219,182 @625,137 delta:1194,477 :569,340 @625,137 delta:1418,659 :793,522 @625,137 delta:628,625 :3,488 @625,137 delta:725,651 :100,514 @625,137 delta:646,177 :21,40 @625,137 delta:622,134 :-3,-3 @625,137 delta:646,178 :21,41 @625,137 delta:1402,179 :777,42 @625,137 delta:908,453 :283,316 @625,137 delta:1104,456 :479,319 @625,137 delta:677,644 :52,507 @625,137 */ PointerInfo info = MouseInfo.getPointerInfo(); Point p = info.getLocation(); Point last_last_last_p = new Point(0,0); Point last_last_p = new Point(0,0); Point last_p = new Point(0,0); Point base_p = new Point(0,0); Point delta_p = new Point(0,0); Point last_pause_p = new Point(0,0); Point last_delta_p = new Point(0,0); int w = 0; int h = 0; int count = 0; int img_count = 0; boolean base_p_set = false; base_p.x = 0; base_p.y = 0; last_p.x = 0; last_p.y = 0; /* move cursor to point , cannot work out how to press so use time */ /* if mouse stops digitise base point */ /* if mouse pauses digitise delta point */ /* to set new base move mouse to left of screen */ while (true) { last_last_last_p.x = last_last_p.x; last_last_p.x = last_p.x; last_p.x = p.x; last_last_last_p.y = last_last_p.y; last_last_p.y = last_p.y; last_p.y = p.y; count ++; //PointerInfo info = MouseInfo.getPointerInfo(); p = info.getLocation(); if ( p.equals(last_last_p) ){ /* mouse paused so capture */ if ( ( p.x == 0 ) || ( p.y == 0 )) { System.out.format("\n /* Move mouse to base, and wait */\n" ); base_p.x = 0; base_p.y = 0; base_p_set = false ; } /* trigger event */ /* set base */ if ( count == 3 ){ if ( base_p_set == false){ base_p_set = true; base_p.x = p.x; base_p.y = p.y; System.out.format("\n/*base_set( %d, %d );*/\n", base_p.x , base_p.y ); } base_p_set = true; /* output delta */ delta_p.x = ( p.x- base_p.x ); delta_p.y = ( p.y- base_p.y ); //System.out.format("\ndelta:, %d, %d\n", delta_p.x , delta_p.y ); //System.out.println( "delta:" + p.x+","+p.y+" :"+delta_p.x + "," + delta_p.y +" @" + base_p.x +"," + base_p.y ); w = (delta_p.x-last_delta_p.x); h = (delta_p.y-last_delta_p.y); System.out.println( " delta(" + p.x+","+p.y+", "+delta_p.x + "," + delta_p.y + ","+ " /*@*/ " + base_p.x +"," + base_p.y +","+ " /*rect*/ " + // img_count + ","+ last_delta_p.x +"," + last_delta_p.y +","+ w + "," + h + "," + ");" ); last_delta_p.x = delta_p.x ; last_delta_p.y = delta_p.y ; w = ( p.x - last_pause_p.x ); h = ( p.y - last_pause_p.y ); System.out.println( " rects[ count++ ] = new Rectangle( " + last_pause_p.x + "," + last_pause_p.y + "," + w + "," + h + " ); " ); last_pause_p.x = p.x; last_pause_p.y = p.y; } /* work out width and height of last rectangle */ w = (delta_p.x-last_delta_p.x); h = (delta_p.y-last_delta_p.y); /* if rect is well formed, take a snapshot */ if ( ( w > 0 ) && ( h > 0 ) ){ /* does not work yet rect = new Rectangle( last_delta_p.x, last_delta_p.y , w , h ); image = robot.createScreenCapture( rect ); saveImage( (fileName+img_cnt) , image ); */ } //System.out.format("*" ); } else { count = 0; } //robot.delay(500); sleep(); } } private static void sleep() { try { Thread.sleep(500); } catch (InterruptedException e) { e.printStackTrace(); } } private static void delta( int x, int y, int delta_x, int delta_y, int base_x, int base_y, int rx,int ry , int w, int h ){ } private static void deltas() { /*base set:, 0, 0 */ delta(0,0, 0,0, /*@*/ 0,0, /*rect*/ 211,-265,-211,265); delta(558,298, 558,298, /*@*/ 0,0, /*rect*/ 0,0,558,298); /* Move mouse to base, and wait Move mouse to base, and wait base set:, 560, 298 */ } } /* Buttons: [] Alt-O Options [] Alt-C Calls Calls:- Alt-J Join Call */ /* Move mouse to base, and wait */