/* * * GrabScreen - Exploration into using Java robot. Try and automate auto join and reconnect * * Use JavaRobot to Grab the screen, * monitor areas for changes and generate events to be serviced by routines. * * To automate the GUI, click on buttons. * If the user moves the mouse, disable the events for a period of time. * * EUAN - End User App Nerd * EUAN - End User App Norman * EUAN - End User App Numpty * * usage: * javac GrabScreen - used to digitise the GUI as seen on the screen. The rectangles where ther buttons are in screen co-ordinates are required. * java GrabScreen filename x y w d - area on screen used to exit the program. * * java GrabScreen eua 100 100 10 10 - if this area changes, end the script. e.g cmd maximised or editor maximised. * * * Inspired by: * https://alvinalexander.com/java/java-robot-class-example-mouse-keystroke * http://www.dougrice.plus.com/hp/Theory/Index.htm * * Possible ideas considered:- * * Grab small area of screen and watch the screen for a Change. * Exit on screen change. * * Extend this to poll multiples areas. * * A simple real time system works by a simple loop. * * init() * while( true ){ * poll_and_make_events * service_events() * sleep() * } * Java robot allows areas of screen to be captured, keys to be pressed, move the mouse and press mouse buttons. e.g. // Set up list of images Rectangle rect = new Rectangle( rx , ry , rw , rh ); BufferedImage firstImage = robot.createScreenCapture( rect ); BufferedImage image = robot.createScreenCapture( rect ); // set up a rectangle to be used to take a snapshot of where the mouse has been moved to. Rectangle rect_m = new Rectangle( rx , ry , rw , rh ); BufferedImage image_m = robot.createScreenCapture( rect_m ); Unfortunately, they are screen co-ordinate and it is difficult to map application GUI co-ordinates to Screen Co-ordinates. poll_and make_events:- monitor many areas and generate events. When small areas of screen change, generate events. Watch icons for change and buttons being disabled. Saves screen capture as .png Also consider analysing a small image captured off the screen We need to process it to classify it. We need to do some image arithmatics. Compare the middle row of pixels between first capture of the image and current image to work out dImage/dt. Other Tests: Is the image blank? For each pixel, are they equal? Is the image complex? For each pixel, work out the mean, standard deviation Is the image complex? Each Pixel is RGB, Is the image dark? For each pixel sum the pixels. What colour? for each PIXEL work out which is max R,G or B and report R G B = " | diffs: " + (r-g) + ", " + (g-b) + ", " + (b-r) * * * * * * * * * This program can grab some of the screen, but does not know what it is. The window is focused, but we assume it is where we think it is. If it moves we have problems. We could check if it is where we think it is with a special test. Look for the edges. Grab a thin letterbox and from the center look for the edges. Grab a thin Portrate and from the center look for the edges. Using these coordinates grab the image of interest. define the areas to watch. wait till an area changes and return. capture before and after screen shots Issues: eua starts in middle of the screen eua is different on Linux and Windows so mouse positions vary across installations Need some deadmans handle if user moves mouse. If the mouse is not used, ensure EUA is focused. If the mouse position moves unexpectedly, disable mouse and keypresses. If the mouse position remains stable, click on EUA titlebar to focus it. Need to disable mouse and key presses if EUA is not in focus. When focused header is different to unfocused. */ /* ==================================== */ import java.io.*; import java.awt.AWTException; import java.awt.Robot; import java.awt.event.InputEvent; import java.awt.event.KeyEvent; /* ==================================== */ import java.awt.Color; import java.awt.Rectangle; import java.awt.image.BufferedImage; /* ==================================== */ import javax.imageio.*; import java.awt.MouseInfo; import java.awt.Point; import java.awt.PointerInfo; /* ==================================== */ //https://docs.oracle.com/javase/7/docs/api/java/awt/GraphicsDevice.html /* ==================================== */ public class GrabScreen { Robot robot = new Robot(); static int lx; static int ly; static int ic = 0 ; static boolean d = true; public static void main(String[] args) throws AWTException { System.out.println( "usage: java GrabsScreen img x y w h - waits for image to change" ); System.out.println( "usage: java GrabsScreen - mouse co-ordinates" ); /* if no parameters show mouse co-ords */ if ( args.length > 4 ) { System.out.println("starting...."); System.out.println("args[0]: " + args[0] ); System.out.println("args[1]: " + args[1] ); System.out.println("args[2]: " + args[2] ); System.out.println("args[3]: " + args[3] ); System.out.println("args[4]: " + args[4] ); new GrabScreen( args[0], Integer.parseInt(args[1]), Integer.parseInt(args[2]), Integer.parseInt(args[3]), Integer.parseInt(args[4]) ); } else { mouseCapture( ); } } public GrabScreen( String fileName , int rx, int ry, int rw, int rh ) throws AWTException { int count =0; int count_images =0; int i ; robot.setAutoDelay(40); robot.setAutoWaitForIdle(true); //PointerInfo PointerInfo info = MouseInfo.getPointerInfo(); Point p = info.getLocation(); Point last_p = new Point(0,0); int safeCounter = 5; // if mouse have moves by use set safeCounter. int safeCounterInit = 5; // if mouse have moves by use set safeCounter. boolean safeMouseMove = false; // if script moves mouse set this Rectangle[] rects = new Rectangle[ 20 ] ; BufferedImage[] firstImages = new BufferedImage[ 20 ]; BufferedImage[] lastImages = new BufferedImage[ 20 ]; BufferedImage[] images = new BufferedImage[ 20 ]; /* set up a rectangle to be used to take a snapshot of where the mouse has been moved to */ Rectangle rect_m = new Rectangle( 0, 0, 10, 10 ); BufferedImage image_m = robot.createScreenCapture( rect_m ); Color cmd; int[] matchRanks = new int[ 20 ]; int[] lastMatchRanks = new int[ 20 ]; boolean[] matchRankFlags = new boolean[ 20 ]; /* Set up list of images */ Rectangle rect = new Rectangle( rx , ry , rw , rh ); BufferedImage firstImage = robot.createScreenCapture( rect ); BufferedImage image = robot.createScreenCapture( rect ); /* save image */ saveImage( fileName , image ); /* options rectangle */ count = 1; /* * * Watch these rectangles on the screen to see if they change. * Make an event out of it, * use a Flag to indicate an ISR "interrupt service routine" needs to be run * Get an ISR interrupt service routine to handle the events. * */ rects[ count++ ] = new Rectangle( 564,327,34,24 ); // 1- option rects[ count++ ] = new Rectangle( 1314,326,34,27 ); // 2- calls rects[ count++ ] = new Rectangle( 668,415,569,288 );// 3- pop-up window rects[ count++ ] = new Rectangle( 946,332,23,18 ); // 4- green rects[ count++ ] = new Rectangle( 778,480,350,155 );// 5- Reconnect Dialogue box rects[ count++ ] = new Rectangle( 653,390,501,19 ); // 6- Options Dialouge titles rects[ count++ ] = new Rectangle( 639,347,627,422 );// 7- Options Dialouge area rects[ count++ ] = new Rectangle( 791,600,116,22 ); // 8- reconnect button rects[ count++ ] = new Rectangle( 907,300,121,15 ); // 9- watch the screen title to see if there is a move of the screen. rects[ count++ ] = new Rectangle( 904,787,104,20 ); // 10 - mute button rects[ count++ ] = new Rectangle( 1252,789,97,22 ); // 11 - keypad button // rects[ count++ ] = new Rectangle( 680,662, 118,26 );// 12 - Join Button rects[ count++ ] = new Rectangle( 1107,661,116,25 );// 13 - cancel Button /* define where to click buttons here */ /*base_set( 554, 294 );*/ Point titleBarHome = new Point( 554+5 , 294+5); // Park mouse at left of title bar - useful to check Registration of the GUI on the screen. Point buttonJoin = new Point( 740 , 670); // JoinButton Point buttonReconnect = new Point( 791+50 , 600+10); // Reconnect Button int x; int y; //count--; System.out.println( "count:" + count ); for( i=1 ; i < count ; i++){ firstImages[i] = robot.createScreenCapture( rects[i] ); lastImages[i] = robot.createScreenCapture( rects[i] ); saveImage( (fileName+"_"+i) , firstImages[i] ); } // int matchRank = testImage( image ); int matchRank = testImage( image, firstImage ); System.out.println( "M:" + matchRank ); //reportImage( image ); System.out.println( "count:" + count ); /* Run a timer to disable key and mouse presses if the mouse position has been moved by the user instead of the script. If the mouse position is stable, then enable keypresses and mouse moves If the script moves the mouse disable triggering this, or shorten the timeout. */ while( matchRank == 0 ){ image = robot.createScreenCapture( rect ); matchRank = testImage( image, firstImage ); /* get pointer position to help */ info = MouseInfo.getPointerInfo(); p = info.getLocation(); if ( ( p.x != last_p.x ) || ( p.y != last_p.y ) ){ /* mouse moved possibly by user, so disable events */ if ( safeMouseMove ){ safeCounter = 0; } else { safeCounter = safeCounterInit; } } safeMouseMove = false; /* caputure the area where the mouse is pointing at */ /* This was to try out an idea of using colour to say where to move to */ /* rect_m = new Rectangle( p.x , p.y , 50 , 10 ); image_m = robot.createScreenCapture( rect_m ); cmd = colourMouse( image_m ); robot.mouseMove( cmd.getRed(), cmd.getGreen() ); */ /* print out the mouse position */ /* consider using pause to generate event to sample the mouse and output rectangle co-ordinates */ System.out.println( "-- !" + safeCounter + " @ " + p.x + "," + p.y + " last:"+ last_p.x + "," + last_p.y +" w:" + (p.x-last_p.x) + ",h:" + (p.y-last_p.y) + " -------------------" ); last_p.x = p.x; last_p.y = p.y; //System.out.println( "M:" + matchRank ); for( i = 1; i < count ; i++){ /* https://docs.oracle.com/javase/7/docs/api/java/awt/image/BufferedImage.html */ /* save image for dImage/dt */ /* how do you copy the objects in java ?*/ //lastImages[ i ] = images[ i ].getSubimage(0,0,images[ i ].getWidth(),images[ i ].getHeight()); // getSubimage(int x, int y, int w, int h) images[ i ] = robot.createScreenCapture( rects[ i ] ); matchRanks[ i ] = testImage( images[ i ], firstImages[ i ] ); //matchRanks[ i ] = testImage( images[ i ], lastImages[ i ] ); /* images are not equal */ /* testImage returns differences from when the program was started to now */ /* take the differential */ if ( ( matchRanks[ i ] - lastMatchRanks[ i ] ) > 0 ){ /* disable event processing if the mouse has been moved by the user, so the user can kill the script */ if( safeCounter == 0 ) { matchRankFlags[ i ] = true; /* Interupt Service Routines have to poll these and reset these flags */ } } lastMatchRanks[ i ] = matchRanks[ i ]; //System.out.println( "("+matchRanks[i] + ":"+lastMatchRanks[i] + " "+matchRankFlags[i] + ") " ); } /* if mouse moved by user safeCounter set to disable events. */ if( safeCounter > 0 ) { /* try and click on GUI to focus it */ if( safeCounter == 1 ) { /*move the mouse around to each rectangle defined to see if any rouge rectanlges defined.*/ for( i = 1; i < count ; i++){ robot.mouseMove( rects[i].x, rects[i].y ); robot.delay(250); }; /* Park the mouse to the top left of the title bar */ robot.mouseMove(titleBarHome.x, titleBarHome.y); robot.delay(500); /* click it to focus the GUI */ leftClick(); robot.delay(500); safeMouseMove = true; } safeCounter--; } robot.delay(500); for( i = 1; i < count ; i++){ if( matchRankFlags[ i ] == true ){ count_images++; matchRankFlags[ i ] = false; System.out.println( "ISR[" +i +"] "); saveImage( (fileName+"_"+i+"_end_"+count_images ) , images[i] ); if( i==1 ){ System.out.println( "ISR[" +i +"] Options.. Called Ended! "); } if ( ( i==2 )||( i==3 ) ){ System.out.println( "ISR[" +i +"] Joining Call... "); /* Join Call - click Join Button*/ x=buttonJoin.x; y=buttonJoin.y; rect_m = new Rectangle( x-25,y-25 , 50 , 50 ); image_m = robot.createScreenCapture( rect_m ); saveImage( ( fileName+"_"+i+"_press_" + count_images ) , image_m ); robot.mouseMove( x,y); robot.delay(500); leftClick(); robot.delay(500); /* we need to monitor the Options icon during a call */ safeMouseMove = true; } if( i==6 ){ System.out.println( "ISR[" +i +"] options titles "); } if( i==7 ){ System.out.println( "ISR[" +i +"] options Dialogue "); } if( i==8 ){ System.out.println( "ISR[" +i +"] reconnecting... "); /* reconnect - click Reconnect Button*/ x=buttonReconnect.x; y=buttonReconnect.y; rect_m = new Rectangle( x-25,y-25 , 50 , 50 ); image_m = robot.createScreenCapture( rect_m ); saveImage( (fileName+"_"+i+"_press_"+count_images ) , image_m ); robot.mouseMove( x,y ); robot.delay(500); leftClick(); robot.delay(500); safeMouseMove = true; } /* This did not work as it reacted to the focus */ if( i==9 ){ /* We need a way to detect when the window has moved */ System.out.println( "ISR[" +i +"] GUI window moved!!! "); } if( i==10 ){ System.out.println( "ISR[" +i +"] Mute Button "); } if( i==11 ){ System.out.println( "ISR[" +i +"] Keypad button "); } } } } for( i=1 ; i < count ; i++){ saveImage( (fileName+"_"+i+"_end") , images[i] ); } System.exit(0); } private int testImage( BufferedImage image, BufferedImage firstImage ) { /* * look at images find middle and compare to firstImage. */ int y = image.getHeight() / 2; Color firstC = new Color( image.getRGB( 0, y )); Color c = new Color( image.getRGB( 0 ,y )); int sumEquals = 0; int sumDeltas = 0; /* * to work out match, * work out the difference between * the row of pixels in the middle of the images * If working out dImage/dt we */ for ( y = 0; y < image.getHeight(); y +=3 ) { for (int x = 0; x < image.getWidth(); x++) { firstC = new Color( firstImage.getRGB( x, y ) ); c = new Color( image.getRGB( x, y ) ); /* work out a statistic */ if ( c.equals( firstC ) ) { sumEquals += 1; } else { sumEquals += -1; }; /* work out dImage/dt */ /* image - firstImage */ if ( !c.equals( firstC ) ) { sumDeltas += 1; }; // System.out.println( x + "," + y + "," + c.equals( firstC ) ); } } //return sumEquals; return sumDeltas; } private Color colourMouse( BufferedImage image ) { /* * look at images find colours and use this to move the mouse to the encoded co-ordinates. */ int y = image.getHeight() / 2; Color cmd = new Color( image.getRGB( 5 , y ) ); Color mx = new Color( image.getRGB( 10 , y ) ); Color my = new Color( image.getRGB( 15 , y ) ); // System.out.println( "cmd: "+ cmd +" mx: " + mx + " my: "+ my ); System.out.print( "cmd: "+ cmd ); return cmd; } private void saveImage( String fileName , BufferedImage image ) { System.out.println( "filename:" + fileName ); try { // retrieve image // File outputfile = new File("saved.png"); File outputfile = new File( fileName + ".png"); ImageIO.write(image, "png", outputfile); } catch (IOException e) { System.out.println("image save failed..." ); } } private void leftClick() { robot.mousePress(InputEvent.BUTTON1_MASK); robot.delay(200); robot.mouseRelease(InputEvent.BUTTON1_MASK); robot.delay(200); } private void type(int i) { robot.delay(10); robot.keyPress(i); robot.keyRelease(i); } private void type(String s) { byte[] bytes = s.getBytes(); for (byte b : bytes) { int code = b; // keycode only handles [A-Z] (which is ASCII decimal [65-90]) if (code > 96 && code < 123) code = code - 32; // robot.delay(40); robot.delay(10); robot.keyPress(code); robot.keyRelease(code); } } private void script() { /* robot.delay(4000); robot.mouseMove(40, 130); robot.delay(500); // select App to bring to the front. leftClick(); robot.delay(500); type("\n\n\n\n"); robot.mouseMove(40, 130); robot.delay(500); leftClick(); robot.delay(500); leftClick(); robot.delay(500); type("Hello, world"); robot.mouseMove(40, 160); robot.delay(500); leftClick(); robot.delay(500); leftClick(); robot.delay(500); type("This is a test of the Java Robot class"); robot.delay(50); type(KeyEvent.VK_DOWN); robot.delay(1000); */ } /************************************************ * * This routine reports the cursor position. * * If the mouse pauses, the co-ordinates are output. * * x,y, rel_x, rel_y, @ , base_x,base_y * * It is possible to set a base point, by moving the cursor to 0,0 * then to the new base point * By moving the cursot to the top left or 0,0, * waiting and then to the new point. * * Output the last x,y,w,h * & *************************************************/ public static int mouseCapture( ) { PointerInfo info = MouseInfo.getPointerInfo(); Point p = info.getLocation(); Point last_last_last_p = new Point(0,0); Point last_last_p = new Point(0,0); Point last_p = new Point(0,0); Point base_p = new Point(0,0); Point delta_p = new Point(0,0); Point last_pause_p = new Point(0,0); Point last_delta_p = new Point(0,0); int w = 0; int h = 0; int count = 0; int img_count = 0; boolean base_p_set = false; base_p.x = 0; base_p.y = 0; last_p.x = 0; last_p.y = 0; /* move cursor to point , cannot work out how to press so use time */ /* if mouse stops digitise base point */ /* if mouse pauses digitise delta point */ /* to set new base move mouse to left of screen */ while (true) { last_last_last_p.x = last_last_p.x; last_last_p.x = last_p.x; last_p.x = p.x; last_last_last_p.y = last_last_p.y; last_last_p.y = last_p.y; last_p.y = p.y; count ++; //PointerInfo info = MouseInfo.getPointerInfo(); p = info.getLocation(); if ( p.equals(last_last_p) ){ /* mouse paused so capture */ if ( ( p.x == 0 ) || ( p.y == 0 )) { System.out.format("\n /* Move mouse to base, and wait */\n" ); base_p.x = 0; base_p.y = 0; base_p_set = false ; } /* trigger event */ /* set base */ if ( count == 3 ){ if ( base_p_set == false){ base_p_set = true; base_p.x = p.x; base_p.y = p.y; System.out.format("\n/*base_set( %d, %d );*/\n", base_p.x , base_p.y ); } base_p_set = true; /* output delta */ delta_p.x = ( p.x- base_p.x ); delta_p.y = ( p.y- base_p.y ); //System.out.format("\ndelta:, %d, %d\n", delta_p.x , delta_p.y ); //System.out.println( "delta:" + p.x+","+p.y+" :"+delta_p.x + "," + delta_p.y +" @" + base_p.x +"," + base_p.y ); w = (delta_p.x-last_delta_p.x); h = (delta_p.y-last_delta_p.y); System.out.println( " delta(" + p.x+","+p.y+", "+delta_p.x + "," + delta_p.y + ","+ " /*@*/ " + base_p.x +"," + base_p.y +","+ " /*rect*/ " + // img_count + ","+ last_delta_p.x +"," + last_delta_p.y +","+ w + "," + h + "," + ");" ); last_delta_p.x = delta_p.x ; last_delta_p.y = delta_p.y ; w = ( p.x - last_pause_p.x ); h = ( p.y - last_pause_p.y ); System.out.println( " rects[ count++ ] = new Rectangle( " + last_pause_p.x + "," + last_pause_p.y + "," + w + "," + h + " ); " ); last_pause_p.x = p.x; last_pause_p.y = p.y; } /* work out width and height of last rectangle */ w = (delta_p.x-last_delta_p.x); h = (delta_p.y-last_delta_p.y); /* if rect is well formed, take a snapshot */ if ( ( w > 0 ) && ( h > 0 ) ){ /* does not work yet rect = new Rectangle( last_delta_p.x, last_delta_p.y , w , h ); image = robot.createScreenCapture( rect ); saveImage( (fileName+img_cnt) , image ); */ } //System.out.format("*" ); } else { count = 0; } //robot.delay(500); sleep(); } } private static void sleep() { try { Thread.sleep(500); } catch (InterruptedException e) { e.printStackTrace(); } } private static void delta( int x, int y, int delta_x, int delta_y, int base_x, int base_y, int rx,int ry , int w, int h ){ } private static void deltas() { } }