From d03baa2d9853dc637abc9f11910f18bac280cb47 Mon Sep 17 00:00:00 2001 From: Jason Stallings Date: Fri, 13 Mar 2026 02:10:00 -0500 Subject: [PATCH 1/8] wip: First pass at image search. --- binding.gyp | 46 ++- index.d.ts | 63 +++- index.js | 149 ++++++++- src/MMPointArray.h | 9 + src/bitmap_find.c | 21 +- src/bitmap_find.h | 9 + src/bmp_io.c | 10 +- src/color_find.c | 14 +- src/color_find.h | 9 + src/io.c | 22 ++ src/io.h | 3 +- src/png_io.c | 6 +- src/png_io.h | 9 + src/robotjs.cc | 819 +++++++++++++++++++++++++++++++++++++++++---- src/screengrab.c | 3 +- 15 files changed, 1077 insertions(+), 115 deletions(-) diff --git a/binding.gyp b/binding.gyp index 165f8175..16c0b472 100644 --- a/binding.gyp +++ b/binding.gyp @@ -1,4 +1,7 @@ { + 'variables': { + 'robotjs_enable_png%': ' + +export interface Point { + x: number + y: number +} + +export interface ScreenPoint { + x: number + y: number +} + +export interface BitmapSearchOptions { + x?: number + y?: number + width?: number + height?: number + tolerance?: number +} + +export type ImageSearchOptions = BitmapSearchOptions + +export interface Image { width: number height: number - image: any + image: Buffer byteWidth: number bitsPerPixel: number bytesPerPixel: number + screenX?: number + screenY?: number + scaleX?: number + scaleY?: number colorAt(x: number, y: number): string + findColor(color: string, options?: ImageSearchOptions): Point | null + findColors(color: string, options?: ImageSearchOptions): Point[] + countColor(color: string, options?: ImageSearchOptions): number + findImage(needle: Image, options?: ImageSearchOptions): Point | null + findImages(needle: Image, options?: ImageSearchOptions): Point[] + countImage(needle: Image, options?: ImageSearchOptions): number + save(path: string): boolean + toScreenPoint(point: Point, target?: { width: number, height: number }): ScreenPoint + click(point: Point, target?: { width: number, height: number }, button?: string, double?: boolean): ScreenPoint + clickImage(target: Image, options?: ImageSearchOptions, button?: string, double?: boolean): Point | null +} + +export declare class Image { + constructor( + width: number, + height: number, + byteWidth: number, + bitsPerPixel: number, + bytesPerPixel: number, + image: Buffer + ) } +export type Bitmap = Image +export declare const Bitmap: typeof Image + export interface Screen { - capture(x?: number, y?: number, width?: number, height?: number): Bitmap + capture(x?: number, y?: number, width?: number, height?: number): Image +} + +export interface ImageModule { + load(path: string): Image + save(bitmap: Image, path: string): boolean + supportsPNG: boolean } export function setKeyboardDelay(ms: number) : void @@ -31,3 +87,4 @@ export function getPixelColor(x: number, y: number): string export function getScreenSize(): { width: number, height: number } export var screen: Screen +export var image: ImageModule diff --git a/index.js b/index.js index c6fb7836..a2e8ecf8 100644 --- a/index.js +++ b/index.js @@ -1,36 +1,159 @@ -var robotjs = require('./build/Release/robotjs.node'); +const robotjs = require('./build/Release/robotjs.node'); module.exports = robotjs; module.exports.screen = {}; +module.exports.image = {}; + +function Bitmap(width, height, byteWidth, bitsPerPixel, bytesPerPixel, image) +{ + // Convenience constructor for when an object is passed (return of native functions). + if (typeof width === 'object' && width !== null) { + this.screenX = width.screenX; + this.screenY = width.screenY; + this.scaleX = width.scaleX; + this.scaleY = width.scaleY; + image = width.image; + bytesPerPixel = width.bytesPerPixel; + bitsPerPixel = width.bitsPerPixel; + byteWidth = width.byteWidth; + height = width.height; + width = width.width; + } -function bitmap(width, height, byteWidth, bitsPerPixel, bytesPerPixel, image) -{ this.width = width; this.height = height; this.byteWidth = byteWidth; this.bitsPerPixel = bitsPerPixel; this.bytesPerPixel = bytesPerPixel; this.image = image; +} - this.colorAt = function(x, y) - { - return robotjs.getColor(this, x, y); - }; +function Image(width, height, byteWidth, bitsPerPixel, bytesPerPixel, image) +{ + Bitmap.call(this, width, height, byteWidth, bitsPerPixel, bytesPerPixel, image); +} + +Image.prototype = Bitmap.prototype; +Image.prototype.constructor = Image; +function getTargetDimensions(target) +{ + if (!target || typeof target !== 'object') { + return { width: 0, height: 0 }; + } + + if (typeof target.width === 'number' && typeof target.height === 'number') { + return { width: target.width, height: target.height }; + } + + return { width: 0, height: 0 }; } +Image.prototype.colorAt = function(x, y) +{ + return robotjs.getColor(this, x, y); +}; + +Image.prototype.findColor = function(color, options) +{ + return robotjs.findColor(this, color, options); +}; + +Image.prototype.findColors = function(color, options) +{ + return robotjs.findColors(this, color, options); +}; + +Image.prototype.countColor = function(color, options) +{ + return robotjs.countColor(this, color, options); +}; + +Image.prototype.findImage = function(needle, options) +{ + return robotjs.findImage(this, needle, options); +}; + +Image.prototype.findImages = function(needle, options) +{ + return robotjs.findImages(this, needle, options); +}; + +Image.prototype.countImage = function(needle, options) +{ + return robotjs.countImage(this, needle, options); +}; + +Image.prototype.save = function(path) +{ + return robotjs.saveImage(this, path); +}; + +Image.prototype.toScreenPoint = function(point, target) +{ + var dimensions = getTargetDimensions(target); + var scaleX = typeof this.scaleX === 'number' && this.scaleX > 0 ? this.scaleX : 1; + var scaleY = typeof this.scaleY === 'number' && this.scaleY > 0 ? this.scaleY : 1; + var screenX = typeof this.screenX === 'number' ? this.screenX : 0; + var screenY = typeof this.screenY === 'number' ? this.screenY : 0; + + return { + x: Math.round(screenX + ((point.x + Math.floor(dimensions.width / 2)) / scaleX)), + y: Math.round(screenY + ((point.y + Math.floor(dimensions.height / 2)) / scaleY)) + }; +}; + +Image.prototype.click = function(point, target, button, double) +{ + var screenPoint = this.toScreenPoint(point, target); + + module.exports.moveMouse(screenPoint.x, screenPoint.y); + if (typeof button === 'undefined') { + module.exports.mouseClick(); + } else if (typeof double === 'undefined') { + module.exports.mouseClick(button); + } else { + module.exports.mouseClick(button, double); + } + + return screenPoint; +}; + +Image.prototype.clickImage = function(target, options, button, double) +{ + var match = this.findImage(target, options); + + if (!match) { + return null; + } + + this.click(match, target, button, double); + return match; +}; + +module.exports.Image = Image; +module.exports.Bitmap = Image; + module.exports.screen.capture = function(x, y, width, height) { //If coords have been passed, use them. if (typeof x !== "undefined" && typeof y !== "undefined" && typeof width !== "undefined" && typeof height !== "undefined") { - b = robotjs.captureScreen(x, y, width, height); - } - else - { - b = robotjs.captureScreen(); + return new Image(robotjs.captureScreen(x, y, width, height)); } - return new bitmap(b.width, b.height, b.byteWidth, b.bitsPerPixel, b.bytesPerPixel, b.image); + return new Image(robotjs.captureScreen()); +}; + +module.exports.image.load = function(path) +{ + return new Image(robotjs.loadImage(path)); +}; + +module.exports.image.save = function(bitmap, path) +{ + return robotjs.saveImage(bitmap, path); }; + +module.exports.image.supportsPNG = !!robotjs.hasPNGSupport; diff --git a/src/MMPointArray.h b/src/MMPointArray.h index 447fc721..44390df8 100644 --- a/src/MMPointArray.h +++ b/src/MMPointArray.h @@ -4,6 +4,11 @@ #include "types.h" +#ifdef __cplusplus +extern "C" +{ +#endif + struct _MMPointArray { MMPoint *array; /* Pointer to actual data. */ size_t count; /* Number of elements in array. */ @@ -30,4 +35,8 @@ void MMPointArrayAppendPoint(MMPointArrayRef pointArray, MMPoint point); /* Set point in array. */ #define MMPointArraySetItem(a, i, item) ((a)->array[i] = item) +#ifdef __cplusplus +} +#endif + #endif /* MMARRAY_H */ diff --git a/src/bitmap_find.c b/src/bitmap_find.c index 7bc83b02..c7b6009d 100644 --- a/src/bitmap_find.c +++ b/src/bitmap_find.c @@ -60,13 +60,14 @@ static int findBitmapInRectAt(MMBitmapRef needle, MMPoint startPoint, UTHashTable *badShiftTable) { - const size_t scanHeight = rect.size.height - needle->height; - const size_t scanWidth = rect.size.width - needle->width; + const size_t scanHeight = rect.origin.y + rect.size.height - needle->height; + const size_t scanWidth = rect.origin.x + rect.size.width - needle->width; MMPoint pointOffset = startPoint; /* const MMPoint lastPoint = MMPointMake(needle->width - 1, needle->height - 1); */ /* Sanity check */ if (needle->height > haystack->height || needle->width > haystack->width || + needle->height > rect.size.height || needle->width > rect.size.width || !MMBitmapRectInBounds(haystack, rect)) { return -1; } @@ -86,8 +87,6 @@ static int findBitmapInRectAt(MMBitmapRef needle, while (pointOffset.x <= scanWidth) { /* Check offset in |haystack| for |needle|. */ if (needleAtOffset(needle, haystack, pointOffset, tolerance)) { - ++pointOffset.x; - ++pointOffset.y; *point = pointOffset; return 0; } @@ -156,7 +155,7 @@ int findBitmapInRect(MMBitmapRef needle, initBadShiftTable(&badShiftTable, needle); ret = findBitmapInRectAt(needle, haystack, point, rect, - tolerance, MMPointZero, &badShiftTable); + tolerance, rect.origin, &badShiftTable); destroyBadShiftTable(&badShiftTable); return ret; } @@ -165,15 +164,15 @@ MMPointArrayRef findAllBitmapInRect(MMBitmapRef needle, MMBitmapRef haystack, MMRect rect, float tolerance) { MMPointArrayRef pointArray = createMMPointArray(0); - MMPoint point = MMPointZero; + MMPoint point = rect.origin; UTHashTable badShiftTable; initBadShiftTable(&badShiftTable, needle); while (findBitmapInRectAt(needle, haystack, &point, rect, tolerance, point, &badShiftTable) == 0) { - const size_t scanWidth = (haystack->width - needle->width) + 1; + const size_t scanWidth = rect.origin.x + (rect.size.width - needle->width) + 1; MMPointArrayAppendPoint(pointArray, point); - ITER_NEXT_POINT(point, scanWidth, 0); + ITER_NEXT_POINT(point, scanWidth, rect.origin.x); } destroyBadShiftTable(&badShiftTable); @@ -184,15 +183,15 @@ size_t countOfBitmapInRect(MMBitmapRef needle, MMBitmapRef haystack, MMRect rect, float tolerance) { size_t count = 0; - MMPoint point = MMPointZero; + MMPoint point = rect.origin; UTHashTable badShiftTable; initBadShiftTable(&badShiftTable, needle); while (findBitmapInRectAt(needle, haystack, &point, rect, tolerance, point, &badShiftTable) == 0) { - const size_t scanWidth = (haystack->width - needle->width) + 1; + const size_t scanWidth = rect.origin.x + (rect.size.width - needle->width) + 1; ++count; - ITER_NEXT_POINT(point, scanWidth, 0); + ITER_NEXT_POINT(point, scanWidth, rect.origin.x); } destroyBadShiftTable(&badShiftTable); diff --git a/src/bitmap_find.h b/src/bitmap_find.h index d077ee52..66c2b3bc 100644 --- a/src/bitmap_find.h +++ b/src/bitmap_find.h @@ -6,6 +6,11 @@ #include "MMBitmap.h" #include "MMPointArray.h" +#ifdef __cplusplus +extern "C" +{ +#endif + /* Convenience wrapper around findBitmapInRect(), where |rect| is the bounds * of |haystack|. */ #define findBitmapInBitmap(needle, haystack, pointPtr, tol) \ @@ -49,4 +54,8 @@ MMPointArrayRef findAllBitmapInRect(MMBitmapRef needle, MMBitmapRef haystack, size_t countOfBitmapInRect(MMBitmapRef needle, MMBitmapRef haystack, MMRect rect, float tolerance); +#ifdef __cplusplus +} +#endif + #endif /* BITMAP_H */ diff --git a/src/bmp_io.c b/src/bmp_io.c index a1c9d0bf..4b16938c 100644 --- a/src/bmp_io.c +++ b/src/bmp_io.c @@ -370,10 +370,12 @@ static uint8_t *readImageData(FILE *fp, size_t width, size_t height, static void copyBGRDataFromMMBitmap(MMBitmapRef bitmap, uint8_t *dest) { - if (MMRGB_IS_BGR && (bitmap->bytewidth % 4) == 0) { /* No conversion needed. */ - memcpy(dest, bitmap->imageBuffer, bitmap->bytewidth * bitmap->height); + const size_t bytewidth = (bitmap->width * bitmap->bytesPerPixel + 3) & ~3; + + if (MMRGB_IS_BGR && bitmap->bytewidth == bytewidth) { /* No conversion needed. */ + memcpy(dest, bitmap->imageBuffer, bytewidth * bitmap->height); } else { /* Convert to RGB with other-than-4-byte alignment. */ - const size_t bytewidth = (bitmap->width * bitmap->bytesPerPixel + 3) & ~3; + const size_t outputBytesPerPixel = 3; size_t y; /* Copy image data row by row. */ @@ -388,7 +390,7 @@ static void copyBGRDataFromMMBitmap(MMBitmapRef bitmap, uint8_t *dest) rowptr[1] = color->green; rowptr[2] = color->red; - rowptr += bitmap->bytesPerPixel; + rowptr += outputBytesPerPixel; } } } diff --git a/src/color_find.c b/src/color_find.c index afff24b1..415c9127 100644 --- a/src/color_find.c +++ b/src/color_find.c @@ -6,11 +6,13 @@ static int findColorInRectAt(MMBitmapRef image, MMRGBHex color, MMPoint *point, MMRect rect, float tolerance, MMPoint startPoint) { + const size_t maxX = rect.origin.x + rect.size.width; + const size_t maxY = rect.origin.y + rect.size.height; MMPoint scan = startPoint; if (!MMBitmapRectInBounds(image, rect)) return -1; - for (; scan.y < rect.size.height; ++scan.y) { - for (; scan.x < rect.size.width; ++scan.x) { + for (; scan.y < maxY; ++scan.y) { + for (; scan.x < maxX; ++scan.x) { MMRGBHex found = MMRGBHexAtPoint(image, scan.x, scan.y); if (MMRGBHexSimilarToColor(color, found, tolerance)) { if (point != NULL) *point = scan; @@ -33,11 +35,11 @@ MMPointArrayRef findAllColorInRect(MMBitmapRef image, MMRGBHex color, MMRect rect, float tolerance) { MMPointArrayRef pointArray = createMMPointArray(0); - MMPoint point = MMPointZero; + MMPoint point = rect.origin; while (findColorInRectAt(image, color, &point, rect, tolerance, point) == 0) { MMPointArrayAppendPoint(pointArray, point); - ITER_NEXT_POINT(point, rect.size.width, rect.origin.x); + ITER_NEXT_POINT(point, rect.origin.x + rect.size.width, rect.origin.x); } return pointArray; @@ -47,10 +49,10 @@ size_t countOfColorsInRect(MMBitmapRef image, MMRGBHex color, MMRect rect, float tolerance) { size_t count = 0; - MMPoint point = MMPointZero; + MMPoint point = rect.origin; while (findColorInRectAt(image, color, &point, rect, tolerance, point) == 0) { - ITER_NEXT_POINT(point, rect.size.width, rect.origin.x); + ITER_NEXT_POINT(point, rect.origin.x + rect.size.width, rect.origin.x); ++count; } diff --git a/src/color_find.h b/src/color_find.h index 3c732f64..0bc16772 100644 --- a/src/color_find.h +++ b/src/color_find.h @@ -5,6 +5,11 @@ #include "MMBitmap.h" #include "MMPointArray.h" +#ifdef __cplusplus +extern "C" +{ +#endif + /* Convenience wrapper around findColorInRect(), where |rect| is the bounds of * the image. */ #define findColorInImage(image, color, pointPtr, tolerance) \ @@ -46,4 +51,8 @@ MMPointArrayRef findAllColorInRect(MMBitmapRef image, MMRGBHex color, size_t countOfColorsInRect(MMBitmapRef image, MMRGBHex color, MMRect rect, float tolerance); +#ifdef __cplusplus +} +#endif + #endif /* COLOR_FIND_H */ diff --git a/src/io.c b/src/io.c index bf863593..d3c74057 100644 --- a/src/io.c +++ b/src/io.c @@ -1,7 +1,13 @@ #include "io.h" #include "os.h" #include "bmp_io.h" +#if !defined(ROBOTJS_HAS_PNG) +#define ROBOTJS_HAS_PNG 0 +#endif + +#if ROBOTJS_HAS_PNG #include "png_io.h" +#endif #include /* For fputs() */ #include /* For strcmp() */ #include /* For tolower() */ @@ -45,7 +51,12 @@ MMBitmapRef newMMBitmapFromFile(const char *path, case kBMPImageType: return newMMBitmapFromBMP(path, err); case kPNGImageType: +#if ROBOTJS_HAS_PNG return newMMBitmapFromPNG(path, err); +#else + if (err != NULL) *err = kMMIOPNGNotSupportedError; + return NULL; +#endif default: if (err != NULL) *err = kMMIOUnsupportedTypeError; return NULL; @@ -60,7 +71,11 @@ int saveMMBitmapToFile(MMBitmapRef bitmap, case kBMPImageType: return saveMMBitmapAsBMP(bitmap, path); case kPNGImageType: +#if ROBOTJS_HAS_PNG return saveMMBitmapAsPNG(bitmap, path); +#else + return -1; +#endif default: return -1; } @@ -72,8 +87,15 @@ const char *MMIOErrorString(MMImageType type, MMIOError error) case kBMPImageType: return MMBMPReadErrorString(error); case kPNGImageType: +#if ROBOTJS_HAS_PNG return MMPNGReadErrorString(error); +#else + return "PNG support is not enabled in this build"; +#endif default: + if (error == kMMIOPNGNotSupportedError) { + return "PNG support is not enabled in this build"; + } return "Unsupported image type"; } } diff --git a/src/io.h b/src/io.h index 7fac20ed..a2158e87 100644 --- a/src/io.h +++ b/src/io.h @@ -20,7 +20,8 @@ enum _MMImageType { typedef uint16_t MMImageType; enum _MMIOError { - kMMIOUnsupportedTypeError = 0 + kMMIOUnsupportedTypeError = 0, + kMMIOPNGNotSupportedError }; typedef uint16_t MMIOError; diff --git a/src/png_io.c b/src/png_io.c index 4bd84278..5d83df71 100644 --- a/src/png_io.c +++ b/src/png_io.c @@ -3,6 +3,7 @@ #include #include /* fopen() */ #include /* malloc/realloc */ +#include /* memcpy() */ #include #if defined(_MSC_VER) @@ -206,7 +207,8 @@ static PNGWriteInfoRef createPNGWriteInfo(MMBitmapRef bitmap) info->row_pointers = png_malloc(info->png_ptr, sizeof(png_byte *) * info->row_count); - if (bitmap->bytesPerPixel == 3) { + if (bitmap->bytesPerPixel == 3 && + bitmap->bytewidth == (bitmap->width * bitmap->bytesPerPixel)) { /* No alpha channel; image data can be copied directly. */ for (y = 0; y < info->row_count; ++y) { info->row_pointers[y] = bitmap->imageBuffer + (bitmap->bytewidth * y); @@ -220,7 +222,7 @@ static PNGWriteInfoRef createPNGWriteInfo(MMBitmapRef bitmap) } else { /* Ignore alpha channel; copy image data row by row. */ const size_t bytesPerPixel = 3; - const size_t bytewidth = ADD_PADDING(bitmap->width * bytesPerPixel); + const size_t bytewidth = bitmap->width * bytesPerPixel; for (y = 0; y < info->row_count; ++y) { png_uint_32 x; diff --git a/src/png_io.h b/src/png_io.h index 939e228c..f9e13c8e 100644 --- a/src/png_io.h +++ b/src/png_io.h @@ -5,6 +5,11 @@ #include "MMBitmap.h" #include "io.h" +#ifdef __cplusplus +extern "C" +{ +#endif + enum _PNGReadError { kPNGGenericError = 0, kPNGReadError, @@ -34,4 +39,8 @@ int saveMMBitmapAsPNG(MMBitmapRef bitmap, const char *path); * Responsibility for free()'ing data is left up to the caller. */ uint8_t *createPNGData(MMBitmapRef bitmap, size_t *len); +#ifdef __cplusplus +} +#endif + #endif /* PNG_IO_H */ diff --git a/src/robotjs.cc b/src/robotjs.cc index 132784db..77b1444f 100644 --- a/src/robotjs.cc +++ b/src/robotjs.cc @@ -1,4 +1,6 @@ #include +#include +#include #include #include "mouse.h" #include "deadbeef_rand.h" @@ -6,12 +8,19 @@ #include "screen.h" #include "screengrab.h" #include "MMBitmap.h" +#include "bitmap_find.h" +#include "color_find.h" +#include "io.h" #include "snprintf.h" #include "microsleep.h" #if defined(USE_X11) #include "xdisplay.h" #endif +#if !defined(ROBOTJS_HAS_PNG) +#define ROBOTJS_HAS_PNG 0 +#endif + //Global delays. int mouseDelay = 10; int keyboardDelay = 10; @@ -735,9 +744,327 @@ void padHex(MMRGBHex color, char* hex) snprintf(hex, 7, "%06x", color); } +static bool multiplySizeT(size_t left, size_t right, size_t *result) +{ + if (left != 0 && right > (SIZE_MAX / left)) { + return false; + } + + *result = left * right; + return true; +} + +static bool parseSizeT(Napi::Env env, Napi::Value value, const char *name, size_t *result) +{ + if (!value.IsNumber()) { + Napi::Error::New(env, std::string(name) + " must be a number.") + .ThrowAsJavaScriptException(); + return false; + } + + double number = value.As().DoubleValue(); + if (number < 0) { + Napi::Error::New(env, std::string(name) + " must be non-negative.") + .ThrowAsJavaScriptException(); + return false; + } + + *result = static_cast(number); + return true; +} + +static bool parseToleranceOption(Napi::Env env, Napi::Object options, float *tolerance) +{ + if (!options.Has("tolerance")) { + return true; + } + + if (!options.Get("tolerance").IsNumber()) { + Napi::Error::New(env, "tolerance must be a number.") + .ThrowAsJavaScriptException(); + return false; + } + + double value = options.Get("tolerance").As().DoubleValue(); + if (value < 0.0 || value > 1.0) { + Napi::Error::New(env, "tolerance must be between 0.0 and 1.0.") + .ThrowAsJavaScriptException(); + return false; + } + + *tolerance = static_cast(value); + return true; +} + +static MMBitmapRef createBorrowedBitmap(Napi::Env env, Napi::Object obj) +{ + size_t width; + size_t height; + size_t byteWidth; + size_t rowBytes; + size_t bufferSize; + size_t parsedBitsPerPixel; + size_t parsedBytesPerPixel; + uint8_t bitsPerPixel; + uint8_t bytesPerPixel; + + if (!obj.Has("width") || !obj.Has("height") || !obj.Has("byteWidth") || + !obj.Has("bitsPerPixel") || !obj.Has("bytesPerPixel") || !obj.Has("image")) { + Napi::Error::New(env, "Bitmap object is missing required properties.") + .ThrowAsJavaScriptException(); + return NULL; + } + + if (!parseSizeT(env, obj.Get("width"), "width", &width) || + !parseSizeT(env, obj.Get("height"), "height", &height) || + !parseSizeT(env, obj.Get("byteWidth"), "byteWidth", &byteWidth) || + !parseSizeT(env, obj.Get("bitsPerPixel"), "bitsPerPixel", &parsedBitsPerPixel) || + !parseSizeT(env, obj.Get("bytesPerPixel"), "bytesPerPixel", &parsedBytesPerPixel)) { + return NULL; + } + + bitsPerPixel = static_cast(parsedBitsPerPixel); + bytesPerPixel = static_cast(parsedBytesPerPixel); + + if (!obj.Get("image").IsBuffer()) { + Napi::Error::New(env, "image must be a Buffer.").ThrowAsJavaScriptException(); + return NULL; + } + + if (width == 0 || height == 0) { + Napi::Error::New(env, "Bitmap width and height must be greater than zero.") + .ThrowAsJavaScriptException(); + return NULL; + } + + if ((bytesPerPixel != 3 && bytesPerPixel != 4) || + bitsPerPixel != (bytesPerPixel * 8)) { + Napi::Error::New(env, "Bitmap must use 24-bit or 32-bit pixels.") + .ThrowAsJavaScriptException(); + return NULL; + } + + if (!multiplySizeT(width, bytesPerPixel, &rowBytes) || byteWidth < rowBytes) { + Napi::Error::New(env, "byteWidth is smaller than the bitmap row size.") + .ThrowAsJavaScriptException(); + return NULL; + } + + if (!multiplySizeT(byteWidth, height, &bufferSize)) { + Napi::Error::New(env, "Bitmap buffer size is too large.") + .ThrowAsJavaScriptException(); + return NULL; + } + + Napi::Buffer image = obj.Get("image").As >(); + if (image.Length() < bufferSize) { + Napi::Error::New(env, "Bitmap image buffer is smaller than byteWidth * height.") + .ThrowAsJavaScriptException(); + return NULL; + } + + MMBitmapRef bitmap = createMMBitmapWithCleanup(image.Data(), + width, + height, + byteWidth, + bitsPerPixel, + bytesPerPixel, + NULL, + NULL); + if (bitmap == NULL) { + Napi::Error::New(env, "Failed to create bitmap.") + .ThrowAsJavaScriptException(); + } + + return bitmap; +} + +static bool parseSearchRect(Napi::Env env, + Napi::Value optionsValue, + MMBitmapRef bitmap, + MMRect *rect, + float *tolerance) +{ + size_t x = 0; + size_t y = 0; + size_t width = bitmap->width; + size_t height = bitmap->height; + bool hasWidth = false; + bool hasHeight = false; + + *tolerance = 0.0f; + + if (optionsValue.IsUndefined() || optionsValue.IsNull()) { + *rect = MMBitmapGetBounds(bitmap); + return true; + } + + if (!optionsValue.IsObject()) { + Napi::Error::New(env, "Search options must be an object.") + .ThrowAsJavaScriptException(); + return false; + } + + Napi::Object options = optionsValue.As(); + if (options.Has("x") && !parseSizeT(env, options.Get("x"), "x", &x)) { + return false; + } + if (options.Has("y") && !parseSizeT(env, options.Get("y"), "y", &y)) { + return false; + } + if (options.Has("width")) { + hasWidth = true; + if (!parseSizeT(env, options.Get("width"), "width", &width)) { + return false; + } + } + if (options.Has("height")) { + hasHeight = true; + if (!parseSizeT(env, options.Get("height"), "height", &height)) { + return false; + } + } + if (!parseToleranceOption(env, options, tolerance)) { + return false; + } + + if (x > bitmap->width || y > bitmap->height) { + Napi::Error::New(env, "Search origin is outside the bitmap bounds.") + .ThrowAsJavaScriptException(); + return false; + } + + if (!hasWidth) { + width = bitmap->width - x; + } + if (!hasHeight) { + height = bitmap->height - y; + } + + *rect = MMRectMake(x, y, width, height); + if (width == 0 || height == 0 || !MMBitmapRectInBounds(bitmap, *rect)) { + Napi::Error::New(env, "Search rect must stay within the bitmap bounds.") + .ThrowAsJavaScriptException(); + return false; + } + + return true; +} + +static bool parseHexColor(Napi::Env env, Napi::Value value, MMRGBHex *color) +{ + std::string hex; + size_t i; + + if (!value.IsString()) { + Napi::Error::New(env, "Color must be a 6-digit hex string.") + .ThrowAsJavaScriptException(); + return false; + } + + hex = value.As().Utf8Value(); + if (!hex.empty() && hex[0] == '#') { + hex.erase(0, 1); + } + + if (hex.length() != 6) { + Napi::Error::New(env, "Color must be a 6-digit hex string.") + .ThrowAsJavaScriptException(); + return false; + } + + for (i = 0; i < hex.length(); ++i) { + if (!isxdigit((unsigned char)hex[i])) { + Napi::Error::New(env, "Color must be a 6-digit hex string.") + .ThrowAsJavaScriptException(); + return false; + } + } + + *color = static_cast(strtoul(hex.c_str(), NULL, 16)); + return true; +} + +static Napi::Object createPointObject(Napi::Env env, MMPoint point) +{ + Napi::Object obj = Napi::Object::New(env); + obj.Set("x", Napi::Number::New(env, point.x)); + obj.Set("y", Napi::Number::New(env, point.y)); + return obj; +} + +static Napi::Array createPointArray(Napi::Env env, MMPointArrayRef points) +{ + Napi::Array array = Napi::Array::New(env, points->count); + size_t i; + + for (i = 0; i < points->count; ++i) { + array.Set(i, createPointObject(env, MMPointArrayGetItem(points, i))); + } + + return array; +} + +static void finalizeBitmap(Napi::Env env, uint8_t *data, MMBitmap *bitmap) +{ + if (bitmap != NULL) { + destroyMMBitmap(bitmap); + } +} + +static Napi::Object createBitmapObject(Napi::Env env, MMBitmapRef bitmap) +{ + size_t bufferSize; + + if (!multiplySizeT(bitmap->bytewidth, bitmap->height, &bufferSize)) { + destroyMMBitmap(bitmap); + Napi::Error::New(env, "Bitmap is too large.") + .ThrowAsJavaScriptException(); + return Napi::Object::New(env); + } + + Napi::Buffer buffer = Napi::Buffer::New(env, + bitmap->imageBuffer, + bufferSize, + finalizeBitmap, + bitmap); + Napi::Object obj = Napi::Object::New(env); + obj.Set("width", Napi::Number::New(env, bitmap->width)); + obj.Set("height", Napi::Number::New(env, bitmap->height)); + obj.Set("byteWidth", Napi::Number::New(env, bitmap->bytewidth)); + obj.Set("bitsPerPixel", Napi::Number::New(env, bitmap->bitsPerPixel)); + obj.Set("bytesPerPixel", Napi::Number::New(env, bitmap->bytesPerPixel)); + obj.Set("image", buffer); + return obj; +} + +static bool parseImageTypeFromPath(Napi::Env env, + const std::string& path, + MMImageType *imageType) +{ + std::string::size_type dotOffset = path.find_last_of('.'); + + if (dotOffset == std::string::npos || dotOffset == path.length() - 1) { + Napi::Error::New(env, "Unsupported image type. Use a .png or .bmp file.") + .ThrowAsJavaScriptException(); + return false; + } + + *imageType = imageTypeFromExtension(path.c_str() + dotOffset + 1); + if (*imageType == kInvalidImageType) { + Napi::Error::New(env, "Unsupported image type. Use a .png or .bmp file.") + .ThrowAsJavaScriptException(); + return false; + } + + return true; +} + Napi::Value getPixelColorWrapper(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); + size_t x; + size_t y; if (info.Length() != 2) { @@ -748,17 +1075,18 @@ return env.Null(); MMBitmapRef bitmap; MMRGBHex color; - double x = info[0].ToNumber().DoubleValue(); - double y = info[1].ToNumber().DoubleValue(); + if (!parseSizeT(env, info[0], "x", &x) || + !parseSizeT(env, info[1], "y", &y)) { + return env.Null(); + } - if (x < 0 || y < 0 || - !pointVisibleOnMainDisplay(MMPointMake(static_cast(x), static_cast(y)))) + if (!pointVisibleOnMainDisplay(MMPointMake(x, y))) { Napi::Error::New(env, "Requested coordinates are outside the main screen's dimensions.").ThrowAsJavaScriptException(); return env.Null(); } - bitmap = copyMMBitmapFromDisplayInRect(MMRectMake(static_cast(x), static_cast(y), 1, 1)); + bitmap = copyMMBitmapFromDisplayInRect(MMRectMake(x, y, 1, 1)); if (bitmap == NULL) { Napi::Error::New(env, "Failed to capture the requested pixel.") .ThrowAsJavaScriptException(); @@ -822,47 +1150,54 @@ Napi::Value setXDisplayNameWrapper(const Napi::CallbackInfo& info) Napi::Value captureScreenWrapper(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); - size_t x; - size_t y; + size_t x = 0; + size_t y = 0; size_t w; size_t h; + MMSize displaySize = getMainDisplaySize(); //If user has provided screen coords, use them! if (info.Length() == 4) { - //TODO: Make sure requested coords are within the screen bounds, or we get a seg fault. - // An error message is much nicer! - - x = info[0].As().Int32Value(); - y = info[1].As().Int32Value(); - w = info[2].As().Int32Value(); - h = info[3].As().Int32Value(); + if (!parseSizeT(env, info[0], "x", &x) || + !parseSizeT(env, info[1], "y", &y) || + !parseSizeT(env, info[2], "width", &w) || + !parseSizeT(env, info[3], "height", &h)) { + return env.Null(); + } } - else + else if (info.Length() == 0) { //We're getting the full screen. - x = 0; - y = 0; - - //Get screen size. - MMSize displaySize = getMainDisplaySize(); w = displaySize.width; h = displaySize.height; } + else + { + Napi::Error::New(env, "Invalid number of arguments.").ThrowAsJavaScriptException(); + return env.Null(); + } - MMBitmapRef bitmap = copyMMBitmapFromDisplayInRect(MMRectMake(x, y, w, h)); - - uint32_t bufferSize = bitmap->bytewidth * bitmap->height; - Napi::Object buffer = Napi::Buffer::Copy(env, (char*)bitmap->imageBuffer, bufferSize); + if (w == 0 || h == 0 || + x > displaySize.width || y > displaySize.height || + w > displaySize.width - x || h > displaySize.height - y) { + Napi::Error::New(env, "Requested capture rect is outside the main screen's dimensions.") + .ThrowAsJavaScriptException(); + return env.Null(); + } - Napi::Object obj = Napi::Object::New(env); - obj.Set("width", Napi::Number::New(env, bitmap->width)); - obj.Set("height", Napi::Number::New(env, bitmap->height)); - obj.Set("byteWidth", Napi::Number::New(env, bitmap->bytewidth)); - obj.Set("bitsPerPixel", Napi::Number::New(env, bitmap->bitsPerPixel)); - obj.Set("bytesPerPixel", Napi::Number::New(env, bitmap->bytesPerPixel)); - obj.Set("image", buffer); + MMBitmapRef bitmap = copyMMBitmapFromDisplayInRect(MMRectMake(x, y, w, h)); + if (bitmap == NULL) { + Napi::Error::New(env, "Failed to capture the requested screen rect.") + .ThrowAsJavaScriptException(); + return env.Null(); + } + Napi::Object obj = createBitmapObject(env, bitmap); + obj.Set("screenX", Napi::Number::New(env, x)); + obj.Set("screenY", Napi::Number::New(env, y)); + obj.Set("scaleX", Napi::Number::New(env, (double)bitmap->width / (double)w)); + obj.Set("scaleY", Napi::Number::New(env, (double)bitmap->height / (double)h)); return obj; } @@ -875,57 +1210,40 @@ Napi::Value captureScreenWrapper(const Napi::CallbackInfo& info) |_| */ -class BMP -{ - public: - size_t width; - size_t height; - size_t byteWidth; - uint8_t bitsPerPixel; - uint8_t bytesPerPixel; - uint8_t *image; -}; - -//Convert object from Javascript to a C++ class (BMP). -BMP buildBMP(Napi::Object obj) -{ - BMP img; - - img.width = obj.Get("width").As().Uint32Value(); - img.height = obj.Get("height").As().Uint32Value(); - img.byteWidth = obj.Get("byteWidth").As().Uint32Value(); - img.bitsPerPixel = obj.Get("bitsPerPixel").As().Uint32Value(); - img.bytesPerPixel = obj.Get("bytesPerPixel").As().Uint32Value(); - - char* buf = obj.Get("image").As>().Data(); - - //Convert the buffer to a uint8_t which createMMBitmap requires. - img.image = (uint8_t *)malloc(img.byteWidth * img.height); - memcpy(img.image, buf, img.byteWidth * img.height); - - return img; - } - Napi::Value getColorWrapper(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); - MMBitmapRef bitmap; MMRGBHex color; + size_t x; + size_t y; - size_t x = info[1].As().Int32Value(); - size_t y = info[2].As().Int32Value(); + if (info.Length() != 3) + { + Napi::Error::New(env, "Invalid number of arguments.").ThrowAsJavaScriptException(); + return env.Null(); + } - //Get our image object from JavaScript. - BMP img = buildBMP(info[0].ToObject()); + if (!info[0].IsObject() || + !parseSizeT(env, info[1], "x", &x) || + !parseSizeT(env, info[2], "y", &y)) { + if (!info[0].IsObject()) { + Napi::Error::New(env, "First argument must be a bitmap object.") + .ThrowAsJavaScriptException(); + } + return env.Null(); + } - //Create the bitmap. - bitmap = createMMBitmap(img.image, img.width, img.height, img.byteWidth, img.bitsPerPixel, img.bytesPerPixel); + MMBitmapRef bitmap = createBorrowedBitmap(env, info[0].As()); + if (bitmap == NULL) { + return env.Null(); + } // Make sure the requested pixel is inside the bitmap. if (!MMBitmapPointInBounds(bitmap, MMPointMake(x, y))) { Napi::Error::New(env, "Requested coordinates are outside the bitmap's dimensions.").ThrowAsJavaScriptException(); -return env.Null(); + destroyMMBitmap(bitmap); + return env.Null(); } color = MMRGBHexAtPoint(bitmap, x, y); @@ -940,6 +1258,340 @@ return env.Null(); } +Napi::Value findColorWrapper(const Napi::CallbackInfo& info) +{ + Napi::Env env = info.Env(); + MMBitmapRef bitmap; + MMRGBHex color; + MMRect rect; + MMPoint point; + float tolerance; + + if (info.Length() < 2 || info.Length() > 3 || !info[0].IsObject()) { + Napi::Error::New(env, "Invalid arguments.").ThrowAsJavaScriptException(); + return env.Null(); + } + + bitmap = createBorrowedBitmap(env, info[0].As()); + if (bitmap == NULL) { + return env.Null(); + } + + if (!parseHexColor(env, info[1], &color) || + !parseSearchRect(env, + info.Length() == 3 ? info[2] : env.Undefined(), + bitmap, + &rect, + &tolerance)) { + destroyMMBitmap(bitmap); + return env.Null(); + } + + if (findColorInRect(bitmap, color, &point, rect, tolerance) != 0) { + destroyMMBitmap(bitmap); + return env.Null(); + } + + destroyMMBitmap(bitmap); + return createPointObject(env, point); +} + +Napi::Value findColorsWrapper(const Napi::CallbackInfo& info) +{ + Napi::Env env = info.Env(); + MMBitmapRef bitmap; + MMRGBHex color; + MMRect rect; + float tolerance; + MMPointArrayRef points; + Napi::Array result; + + if (info.Length() < 2 || info.Length() > 3 || !info[0].IsObject()) { + Napi::Error::New(env, "Invalid arguments.").ThrowAsJavaScriptException(); + return env.Null(); + } + + bitmap = createBorrowedBitmap(env, info[0].As()); + if (bitmap == NULL) { + return env.Null(); + } + + if (!parseHexColor(env, info[1], &color) || + !parseSearchRect(env, + info.Length() == 3 ? info[2] : env.Undefined(), + bitmap, + &rect, + &tolerance)) { + destroyMMBitmap(bitmap); + return env.Null(); + } + + points = findAllColorInRect(bitmap, color, rect, tolerance); + if (points == NULL) { + destroyMMBitmap(bitmap); + Napi::Error::New(env, "Failed to search bitmap colors.") + .ThrowAsJavaScriptException(); + return env.Null(); + } + + result = createPointArray(env, points); + destroyMMPointArray(points); + destroyMMBitmap(bitmap); + return result; +} + +Napi::Value countColorWrapper(const Napi::CallbackInfo& info) +{ + Napi::Env env = info.Env(); + MMBitmapRef bitmap; + MMRGBHex color; + MMRect rect; + float tolerance; + size_t count; + + if (info.Length() < 2 || info.Length() > 3 || !info[0].IsObject()) { + Napi::Error::New(env, "Invalid arguments.").ThrowAsJavaScriptException(); + return env.Null(); + } + + bitmap = createBorrowedBitmap(env, info[0].As()); + if (bitmap == NULL) { + return env.Null(); + } + + if (!parseHexColor(env, info[1], &color) || + !parseSearchRect(env, + info.Length() == 3 ? info[2] : env.Undefined(), + bitmap, + &rect, + &tolerance)) { + destroyMMBitmap(bitmap); + return env.Null(); + } + + count = countOfColorsInRect(bitmap, color, rect, tolerance); + destroyMMBitmap(bitmap); + return Napi::Number::New(env, count); +} + +Napi::Value findBitmapWrapper(const Napi::CallbackInfo& info) +{ + Napi::Env env = info.Env(); + MMBitmapRef haystack; + MMBitmapRef needle; + MMRect rect; + MMPoint point; + float tolerance; + + if (info.Length() < 2 || info.Length() > 3 || + !info[0].IsObject() || !info[1].IsObject()) { + Napi::Error::New(env, "Invalid arguments.").ThrowAsJavaScriptException(); + return env.Null(); + } + + haystack = createBorrowedBitmap(env, info[0].As()); + if (haystack == NULL) { + return env.Null(); + } + + needle = createBorrowedBitmap(env, info[1].As()); + if (needle == NULL) { + destroyMMBitmap(haystack); + return env.Null(); + } + + if (!parseSearchRect(env, + info.Length() == 3 ? info[2] : env.Undefined(), + haystack, + &rect, + &tolerance)) { + destroyMMBitmap(needle); + destroyMMBitmap(haystack); + return env.Null(); + } + + if (findBitmapInRect(needle, haystack, &point, rect, tolerance) != 0) { + destroyMMBitmap(needle); + destroyMMBitmap(haystack); + return env.Null(); + } + + destroyMMBitmap(needle); + destroyMMBitmap(haystack); + return createPointObject(env, point); +} + +Napi::Value findBitmapsWrapper(const Napi::CallbackInfo& info) +{ + Napi::Env env = info.Env(); + MMBitmapRef haystack; + MMBitmapRef needle; + MMRect rect; + float tolerance; + MMPointArrayRef points; + Napi::Array result; + + if (info.Length() < 2 || info.Length() > 3 || + !info[0].IsObject() || !info[1].IsObject()) { + Napi::Error::New(env, "Invalid arguments.").ThrowAsJavaScriptException(); + return env.Null(); + } + + haystack = createBorrowedBitmap(env, info[0].As()); + if (haystack == NULL) { + return env.Null(); + } + + needle = createBorrowedBitmap(env, info[1].As()); + if (needle == NULL) { + destroyMMBitmap(haystack); + return env.Null(); + } + + if (!parseSearchRect(env, + info.Length() == 3 ? info[2] : env.Undefined(), + haystack, + &rect, + &tolerance)) { + destroyMMBitmap(needle); + destroyMMBitmap(haystack); + return env.Null(); + } + + points = findAllBitmapInRect(needle, haystack, rect, tolerance); + if (points == NULL) { + destroyMMBitmap(needle); + destroyMMBitmap(haystack); + Napi::Error::New(env, "Failed to search for bitmaps.") + .ThrowAsJavaScriptException(); + return env.Null(); + } + + result = createPointArray(env, points); + destroyMMPointArray(points); + destroyMMBitmap(needle); + destroyMMBitmap(haystack); + return result; +} + +Napi::Value countBitmapWrapper(const Napi::CallbackInfo& info) +{ + Napi::Env env = info.Env(); + MMBitmapRef haystack; + MMBitmapRef needle; + MMRect rect; + float tolerance; + size_t count; + + if (info.Length() < 2 || info.Length() > 3 || + !info[0].IsObject() || !info[1].IsObject()) { + Napi::Error::New(env, "Invalid arguments.").ThrowAsJavaScriptException(); + return env.Null(); + } + + haystack = createBorrowedBitmap(env, info[0].As()); + if (haystack == NULL) { + return env.Null(); + } + + needle = createBorrowedBitmap(env, info[1].As()); + if (needle == NULL) { + destroyMMBitmap(haystack); + return env.Null(); + } + + if (!parseSearchRect(env, + info.Length() == 3 ? info[2] : env.Undefined(), + haystack, + &rect, + &tolerance)) { + destroyMMBitmap(needle); + destroyMMBitmap(haystack); + return env.Null(); + } + + count = countOfBitmapInRect(needle, haystack, rect, tolerance); + destroyMMBitmap(needle); + destroyMMBitmap(haystack); + return Napi::Number::New(env, count); +} + +Napi::Value loadImageWrapper(const Napi::CallbackInfo& info) +{ + Napi::Env env = info.Env(); + std::string path; + MMImageType imageType; + MMIOError error = kMMIOUnsupportedTypeError; + MMBitmapRef bitmap; + + if (info.Length() != 1 || !info[0].IsString()) { + Napi::Error::New(env, "loadImage expects a single file path string.") + .ThrowAsJavaScriptException(); + return env.Null(); + } + + path = info[0].As().Utf8Value(); + if (!parseImageTypeFromPath(env, path, &imageType)) { + return env.Null(); + } + + bitmap = newMMBitmapFromFile(path.c_str(), imageType, &error); + if (bitmap == NULL) { + const char *message = MMIOErrorString(imageType, error); + if (message == NULL) { + message = "Failed to load image."; + } + Napi::Error::New(env, message).ThrowAsJavaScriptException(); + return env.Null(); + } + + return createBitmapObject(env, bitmap); +} + +Napi::Value saveImageWrapper(const Napi::CallbackInfo& info) +{ + Napi::Env env = info.Env(); + std::string path; + MMImageType imageType; + MMBitmapRef bitmap; + + if (info.Length() != 2 || !info[0].IsObject() || !info[1].IsString()) { + Napi::Error::New(env, "saveImage expects a bitmap and a file path string.") + .ThrowAsJavaScriptException(); + return env.Null(); + } + + bitmap = createBorrowedBitmap(env, info[0].As()); + if (bitmap == NULL) { + return env.Null(); + } + + path = info[1].As().Utf8Value(); + if (!parseImageTypeFromPath(env, path, &imageType)) { + destroyMMBitmap(bitmap); + return env.Null(); + } + +#if !ROBOTJS_HAS_PNG + if (imageType == kPNGImageType) { + destroyMMBitmap(bitmap); + Napi::Error::New(env, "PNG support is not enabled in this build.") + .ThrowAsJavaScriptException(); + return env.Null(); + } +#endif + + if (saveMMBitmapToFile(bitmap, path.c_str(), imageType) != 0) { + destroyMMBitmap(bitmap); + Napi::Error::New(env, "Failed to save image.") + .ThrowAsJavaScriptException(); + return env.Null(); + } + + destroyMMBitmap(bitmap); + return Napi::Boolean::New(env, true); +} + Napi::Object InitAll(Napi::Env env, Napi::Object exports) { exports.Set(Napi::String::New(env, "dragMouse"), @@ -999,6 +1651,33 @@ Napi::Object InitAll(Napi::Env env, Napi::Object exports) exports.Set(Napi::String::New(env, "getColor"), Napi::Function::New(env, getColorWrapper)); + exports.Set(Napi::String::New(env, "findColor"), + Napi::Function::New(env, findColorWrapper)); + + exports.Set(Napi::String::New(env, "findColors"), + Napi::Function::New(env, findColorsWrapper)); + + exports.Set(Napi::String::New(env, "countColor"), + Napi::Function::New(env, countColorWrapper)); + + exports.Set(Napi::String::New(env, "findImage"), + Napi::Function::New(env, findBitmapWrapper)); + + exports.Set(Napi::String::New(env, "findImages"), + Napi::Function::New(env, findBitmapsWrapper)); + + exports.Set(Napi::String::New(env, "countImage"), + Napi::Function::New(env, countBitmapWrapper)); + + exports.Set(Napi::String::New(env, "loadImage"), + Napi::Function::New(env, loadImageWrapper)); + + exports.Set(Napi::String::New(env, "saveImage"), + Napi::Function::New(env, saveImageWrapper)); + + exports.Set(Napi::String::New(env, "hasPNGSupport"), + Napi::Boolean::New(env, ROBOTJS_HAS_PNG != 0)); + exports.Set(Napi::String::New(env, "getXDisplayName"), Napi::Function::New(env, getXDisplayNameWrapper)); diff --git a/src/screengrab.c b/src/screengrab.c index 5566b370..7e1ba3f3 100644 --- a/src/screengrab.c +++ b/src/screengrab.c @@ -15,8 +15,7 @@ #include #endif -#if defined(IS_MACOSX) -#elif defined(IS_WINDOWS) +#if defined(IS_WINDOWS) static void destroyMMBitmapWindowsDIB(char *bitmapBuffer, void *hint) { if (hint != NULL) { From 15c63e4a8d82e70cec73fbc2b0cebe5a7f29d723 Mon Sep 17 00:00:00 2001 From: Jason Stallings Date: Fri, 13 Mar 2026 02:21:32 -0500 Subject: [PATCH 2/8] wip: Move subtraction after sanity check. --- src/bitmap_find.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/bitmap_find.c b/src/bitmap_find.c index c7b6009d..60b3dca9 100644 --- a/src/bitmap_find.c +++ b/src/bitmap_find.c @@ -60,8 +60,8 @@ static int findBitmapInRectAt(MMBitmapRef needle, MMPoint startPoint, UTHashTable *badShiftTable) { - const size_t scanHeight = rect.origin.y + rect.size.height - needle->height; - const size_t scanWidth = rect.origin.x + rect.size.width - needle->width; + size_t scanHeight; + size_t scanWidth; MMPoint pointOffset = startPoint; /* const MMPoint lastPoint = MMPointMake(needle->width - 1, needle->height - 1); */ @@ -72,6 +72,9 @@ static int findBitmapInRectAt(MMBitmapRef needle, return -1; } + scanHeight = rect.origin.y + rect.size.height - needle->height; + scanWidth = rect.origin.x + rect.size.width - needle->width; + assert(point != NULL); assert(needle != NULL); assert(needle->height > 0 && needle->width > 0); From f11971cc61d2651cc13c04cdb01ed6e88486fd5c Mon Sep 17 00:00:00 2001 From: Jason Stallings Date: Fri, 13 Mar 2026 03:43:19 -0500 Subject: [PATCH 3/8] wip: Fix bug when ITER_NEXT_POINT wrapped back to start_x. --- test/image.js | 279 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 test/image.js diff --git a/test/image.js b/test/image.js new file mode 100644 index 00000000..13af5104 --- /dev/null +++ b/test/image.js @@ -0,0 +1,279 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const robot = require('..'); + +function writeColor(buffer, byteWidth, bytesPerPixel, x, y, hex) { + const offset = (y * byteWidth) + (x * bytesPerPixel); + const color = parseInt(hex, 16); + + buffer[offset] = color & 0xFF; + buffer[offset + 1] = (color >> 8) & 0xFF; + buffer[offset + 2] = (color >> 16) & 0xFF; + if (bytesPerPixel === 4) { + buffer[offset + 3] = 0xFF; + } +} + +function makeBitmap(rows, options) { + options = options || {}; + + const width = rows[0].length; + const height = rows.length; + const bytesPerPixel = options.bytesPerPixel || 4; + const byteWidth = options.byteWidth || (width * bytesPerPixel); + const image = Buffer.alloc(byteWidth * height, 0x00); + + rows.forEach(function(row, y) { + row.forEach(function(hex, x) { + writeColor(image, byteWidth, bytesPerPixel, x, y, hex); + }); + }); + + return new robot.Image(width, height, byteWidth, bytesPerPixel * 8, bytesPerPixel, image); +} + +function create24BitBMP(rows) { + const width = rows[0].length; + const height = rows.length; + const rowStride = (width * 3 + 3) & ~3; + const imageSize = rowStride * height; + const fileSize = 54 + imageSize; + const buffer = Buffer.alloc(fileSize, 0x00); + + buffer.writeUInt16LE(0x4D42, 0); + buffer.writeUInt32LE(fileSize, 2); + buffer.writeUInt32LE(54, 10); + + buffer.writeUInt32LE(40, 14); + buffer.writeInt32LE(width, 18); + buffer.writeInt32LE(height, 22); + buffer.writeUInt16LE(1, 26); + buffer.writeUInt16LE(24, 28); + buffer.writeUInt32LE(0, 30); + buffer.writeUInt32LE(imageSize, 34); + + rows.slice().reverse().forEach(function(row, rowIndex) { + const rowOffset = 54 + (rowIndex * rowStride); + + row.forEach(function(hex, x) { + const color = parseInt(hex, 16); + const pixelOffset = rowOffset + (x * 3); + + buffer[pixelOffset] = color & 0xFF; + buffer[pixelOffset + 1] = (color >> 8) & 0xFF; + buffer[pixelOffset + 2] = (color >> 16) & 0xFF; + }); + }); + + return buffer; +} + +describe('Image', function() { + it('Loads a BMP file and uses it as a search bitmap.', function() { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'robotjs-image-')); + const needlePath = path.join(tmpDir, 'needle.bmp'); + const needleRows = [ + ['ff0000', '0000ff'], + ['00ff00', 'ffff00'] + ]; + const haystack = makeBitmap([ + ['111111', '111111', '111111', '111111'], + ['111111', 'ff0000', '0000ff', '111111'], + ['111111', '00ff00', 'ffff00', '111111'] + ], { byteWidth: 20 }); + let needle; + + fs.writeFileSync(needlePath, create24BitBMP(needleRows)); + + try { + needle = robot.image.load(needlePath); + expect(needle.width).toEqual(2); + expect(needle.height).toEqual(2); + expect(needle.colorAt(0, 0)).toEqual('ff0000'); + expect(needle.colorAt(1, 1)).toEqual('ffff00'); + expect(haystack.findImage(needle)).toEqual({ x: 1, y: 1 }); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it('Rejects unsupported image extensions.', function() { + expect(function() { + robot.image.load('/tmp/not-an-image.txt'); + }).toThrowError(/Unsupported image type/); + }); + + it('Saves and reloads BMP bitmaps.', function() { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'robotjs-image-save-')); + const bmpPath = path.join(tmpDir, 'saved.bmp'); + const bitmap = makeBitmap([ + ['112233', '445566'], + ['778899', 'aabbcc'] + ], { byteWidth: 12, bytesPerPixel: 3 }); + let bmpReloaded; + + try { + expect(robot.image.save(bitmap, bmpPath)).toBe(true); + expect(fs.existsSync(bmpPath)).toBe(true); + + bmpReloaded = robot.image.load(bmpPath); + + expect(bmpReloaded.colorAt(0, 0)).toEqual('112233'); + expect(bmpReloaded.colorAt(1, 1)).toEqual('aabbcc'); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it('Saves and reloads PNG bitmaps when PNG support is enabled.', function() { + let tmpDir, pngPath, bitmap, pngReloaded; + + if (!robot.image.supportsPNG) { + pending('PNG support is not enabled in this build.'); + } + + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'robotjs-image-png-')); + pngPath = path.join(tmpDir, 'saved.png'); + bitmap = makeBitmap([ + ['112233', '445566'], + ['778899', 'aabbcc'] + ], { byteWidth: 12, bytesPerPixel: 3 }); + + try { + expect(bitmap.save(pngPath)).toBe(true); + expect(fs.existsSync(pngPath)).toBe(true); + + pngReloaded = robot.image.load(pngPath); + expect(pngReloaded.colorAt(0, 1)).toEqual('778899'); + expect(pngReloaded.colorAt(1, 0)).toEqual('445566'); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it('Rejects PNG operations when PNG support is disabled.', function() { + const bitmap = makeBitmap([ + ['123456'] + ]); + + if (robot.image.supportsPNG) { + pending('PNG support is enabled in this build.'); + } + + expect(function() { + robot.image.load('/tmp/example.png'); + }).toThrowError(/PNG support is not enabled/); + + expect(function() { + bitmap.save('/tmp/example.png'); + }).toThrowError(/PNG support is not enabled/); + }); + + it('Rejects unsupported save extensions.', function() { + const bitmap = makeBitmap([ + ['123456'] + ]); + + expect(function() { + robot.image.save(bitmap, '/tmp/not-an-image.txt'); + }).toThrowError(/Unsupported image type/); + }); + + it('Converts capture-space points into screen-space points.', function() { + const capture = makeBitmap([ + ['123456', 'abcdef'] + ]); + const needle = makeBitmap([ + ['ffffff', '000000'], + ['000000', 'ffffff'] + ]); + + capture.screenX = 100; + capture.screenY = 200; + capture.scaleX = 2; + capture.scaleY = 2; + + expect(capture.toScreenPoint({ x: 20, y: 10 })).toEqual({ x: 110, y: 205 }); + expect(capture.toScreenPoint({ x: 20, y: 10 }, needle)).toEqual({ x: 111, y: 206 }); + }); + + it('findColors iterates across row boundaries correctly.', function() { + // Regression: ITER_NEXT_POINT previously hardcoded the letiable name + // "point" instead of using its macro parameter. This test places + // matching pixels on separate rows so the macro must increment y + // when x wraps past the row end. + const bitmap = makeBitmap([ + ['ff0000', '111111', '111111'], + ['111111', '111111', 'ff0000'], + ['111111', 'ff0000', '111111'] + ]); + + const results = bitmap.findColors('ff0000'); + expect(results.length).toEqual(3); + expect(results).toEqual([ + { x: 0, y: 0 }, + { x: 2, y: 1 }, + { x: 1, y: 2 } + ]); + }); + + it('findImages iterates across row boundaries correctly.', function() { + // Same regression coverage for the bitmap search path. + const needle = makeBitmap([ + ['aabbcc'] + ]); + const haystack = makeBitmap([ + ['aabbcc', '111111', '111111'], + ['111111', '111111', 'aabbcc'], + ['111111', 'aabbcc', '111111'] + ]); + + const results = haystack.findImages(needle); + expect(results.length).toEqual(3); + expect(results).toEqual([ + { x: 0, y: 0 }, + { x: 2, y: 1 }, + { x: 1, y: 2 } + ]); + }); + + it('Clicks a found image using screen-space coordinates.', function() { + const capture = makeBitmap([ + ['111111', '111111', '111111', '111111'], + ['111111', 'ff0000', '0000ff', '111111'], + ['111111', '00ff00', 'ffff00', '111111'] + ], { byteWidth: 20 }); + const needle = makeBitmap([ + ['ff0000', '0000ff'], + ['00ff00', 'ffff00'] + ], { byteWidth: 12 }); + const originalMoveMouse = robot.moveMouse; + const originalMouseClick = robot.mouseClick; + let movedTo; + let clickedWith; + let match; + + capture.screenX = 10; + capture.screenY = 20; + capture.scaleX = 2; + capture.scaleY = 2; + + robot.moveMouse = function(x, y) { + movedTo = { x: x, y: y }; + }; + robot.mouseClick = function(button, double) { + clickedWith = { button: button, double: double }; + }; + + try { + match = capture.clickImage(needle, { tolerance: 0 }, 'left', true); + expect(match).toEqual({ x: 1, y: 1 }); + expect(movedTo).toEqual({ x: 11, y: 21 }); + expect(clickedWith).toEqual({ button: 'left', double: true }); + } finally { + robot.moveMouse = originalMoveMouse; + robot.mouseClick = originalMouseClick; + } + }); +}); From 8053e5fdd7e1dfaaf869438bc7ab9849b5e0f335 Mon Sep 17 00:00:00 2001 From: Jason Stallings Date: Fri, 13 Mar 2026 03:46:53 -0500 Subject: [PATCH 4/8] fix: Improve reliability of typing on macOS. This is a temporary patch, I need to come back and figure out of delays between down and up keypresses is more reliable. --- src/robotjs.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/robotjs.cc b/src/robotjs.cc index 77b1444f..057e0b53 100644 --- a/src/robotjs.cc +++ b/src/robotjs.cc @@ -25,6 +25,10 @@ int mouseDelay = 10; int keyboardDelay = 10; +#if defined(IS_MACOSX) +static const unsigned MACOS_TYPE_STRING_DEFAULT_CPM = 1000; +#endif + /* __ __ | \/ | ___ _ _ ___ ___ @@ -678,7 +682,11 @@ Napi::Value typeStringWrapper(const Napi::CallbackInfo& info) s = str.c_str(); - typeStringDelayed(s, 0); + #if defined(IS_MACOSX) + typeStringDelayed(s, MACOS_TYPE_STRING_DEFAULT_CPM); + #else + typeStringDelayed(s, 0); + #endif return Napi::Number::New(env, 1); } else { From cf979461bbd421188afab972aced560e61db29ca Mon Sep 17 00:00:00 2001 From: Jason Stallings Date: Sat, 14 Mar 2026 09:30:47 -0500 Subject: [PATCH 5/8] test: Additional bitmap tests. --- test/bitmap.js | 85 ++++++++++++++++++++++++++++++++++-------- test/helpers/images.js | 69 ++++++++++++++++++++++++++++++++++ test/image.js | 67 +-------------------------------- 3 files changed, 139 insertions(+), 82 deletions(-) create mode 100644 test/helpers/images.js diff --git a/test/bitmap.js b/test/bitmap.js index cf04f8fd..4a101dcb 100644 --- a/test/bitmap.js +++ b/test/bitmap.js @@ -1,7 +1,8 @@ -var robot = require('..'); +const robot = require('..'); +const makeBitmap = require('./helpers/images').makeBitmap; describe('Bitmap', () => { - var params = { + const params = { 'width': 'number', 'height': 'number', 'byteWidth': 'number', @@ -11,9 +12,9 @@ describe('Bitmap', () => { }; it('Get a bitmap and check the parameters.', function() { - var img = robot.screen.capture(); + const img = robot.screen.capture(); - for (var x in params) + for (const x in params) { expect(typeof img[x]).toEqual(params[x]); } @@ -21,30 +22,30 @@ describe('Bitmap', () => { it('Get a bitmap of a specific size.', function() { - var size = 10; - var img = robot.screen.capture(0, 0, size, size); + const size = 10; + const img = robot.screen.capture(0, 0, size, size); // Support for higher density screens. - var multi = img.width / size; - var size = size * multi; - expect(img.height).toEqual(size); - expect(img.width).toEqual(size); + const multi = img.width / size; + const scaledSize = size * multi; + expect(img.height).toEqual(scaledSize); + expect(img.width).toEqual(scaledSize); }); it('Get a bitmap and make sure the colorAt works as expected.', function() { - var img = robot.screen.capture(); - var hex = img.colorAt(0, 0); + const img = robot.screen.capture(); + const hex = img.colorAt(0, 0); // t.ok(.it(hex), "colorAt returned valid hex."); expect(hex).toMatch(/^[0-9A-F]{6}$/i); - var screenSize = robot.getScreenSize(); - var width = screenSize.width; - var height = screenSize.height; + const screenSize = robot.getScreenSize(); + let width = screenSize.width; + let height = screenSize.height; // Support for higher density screens. - var multi = img.width / width; + const multi = img.width / width; width = width * multi; height = height * multi; expect(() => img.colorAt(0, height)).toThrowError(/are outside the bitmap/); @@ -53,4 +54,56 @@ describe('Bitmap', () => { expect(() => img.colorAt(9999999999999, 0)).toThrowError(/are outside the bitmap/); expect(() => img.colorAt(0, 9999999999999)).toThrowError(/are outside the bitmap/); }); + + it('Reads padded rows without crossing into row padding.', function() { + const img = makeBitmap([ + ['ff0000', '00ff00', '0000ff'], + ['ffff00', 'ff00ff', '00ffff'] + ], { byteWidth: 16, fill: 0xEE }); + + expect(img.colorAt(0, 0)).toEqual('ff0000'); + expect(img.colorAt(1, 0)).toEqual('00ff00'); + expect(img.colorAt(2, 0)).toEqual('0000ff'); + expect(img.colorAt(0, 1)).toEqual('ffff00'); + expect(img.colorAt(1, 1)).toEqual('ff00ff'); + expect(img.colorAt(2, 1)).toEqual('00ffff'); + }); + + it('Finds colors inside sub-rectangles with non-zero origins.', function() { + const img = makeBitmap([ + ['111111', '222222', '333333', '444444'], + ['555555', '666666', '777777', 'ff00ff'], + ['888888', 'ff00ff', '999999', 'aaaaaa'] + ], { byteWidth: 20, fill: 0xEE }); + + expect(img.findColor('ff00ff')).toEqual({ x: 3, y: 1 }); + expect(img.findColor('ff00ff', { x: 1, y: 2, width: 1, height: 1 })).toEqual({ x: 1, y: 2 }); + expect(img.findColors('ff00ff', { x: 1, y: 1, width: 3, height: 2 })).toEqual([ + { x: 3, y: 1 }, + { x: 1, y: 2 } + ]); + expect(img.countColor('ff00ff', { x: 1, y: 1, width: 3, height: 2 })).toEqual(2); + expect(img.findColor('abcdef')).toBeNull(); + }); + + it('Finds bitmaps inside padded haystacks and respects search rect origins.', function() { + const haystack = makeBitmap([ + ['101010', '101010', '101010', '101010', '101010', '101010'], + ['101010', 'ff0000', '0000ff', 'ff0000', '0000ff', '101010'], + ['101010', '00ff00', 'ffff00', '00ff00', 'ffff00', '101010'], + ['101010', '101010', '101010', '101010', '101010', '101010'] + ], { byteWidth: 28, fill: 0xEE }); + const needle = makeBitmap([ + ['ff0000', '0000ff'], + ['00ff00', 'ffff00'] + ], { byteWidth: 12, fill: 0xEE }); + + expect(haystack.findImage(needle)).toEqual({ x: 1, y: 1 }); + expect(haystack.findImages(needle, { x: 1, y: 1, width: 4, height: 2 })).toEqual([ + { x: 1, y: 1 }, + { x: 3, y: 1 } + ]); + expect(haystack.countImage(needle, { x: 1, y: 1, width: 4, height: 2 })).toEqual(2); + expect(haystack.findImage(needle, { x: 0, y: 0, width: 1, height: 4 })).toBeNull(); + }); }); diff --git a/test/helpers/images.js b/test/helpers/images.js new file mode 100644 index 00000000..c8ff9a0f --- /dev/null +++ b/test/helpers/images.js @@ -0,0 +1,69 @@ +const robot = require('../..'); + +function writeColor(buffer, byteWidth, bytesPerPixel, x, y, hex) { + const offset = (y * byteWidth) + (x * bytesPerPixel); + const color = parseInt(hex, 16); + + buffer[offset] = color & 0xFF; + buffer[offset + 1] = (color >> 8) & 0xFF; + buffer[offset + 2] = (color >> 16) & 0xFF; + if (bytesPerPixel === 4) { + buffer[offset + 3] = 0xFF; + } +} + +function makeBitmap(rows, options) { + options = options || {}; + + const width = rows[0].length; + const height = rows.length; + const bytesPerPixel = options.bytesPerPixel || 4; + const byteWidth = options.byteWidth || (width * bytesPerPixel); + const fill = typeof options.fill === 'number' ? options.fill : 0x00; + const image = Buffer.alloc(byteWidth * height, fill); + + rows.forEach(function(row, y) { + row.forEach(function(hex, x) { + writeColor(image, byteWidth, bytesPerPixel, x, y, hex); + }); + }); + + return new robot.Image(width, height, byteWidth, bytesPerPixel * 8, bytesPerPixel, image); +} + +function create24BitBMP(rows) { + const width = rows[0].length; + const height = rows.length; + const rowStride = (width * 3 + 3) & ~3; + const imageSize = rowStride * height; + const fileSize = 54 + imageSize; + const buffer = Buffer.alloc(fileSize, 0x00); + + buffer.writeUInt16LE(0x4D42, 0); + buffer.writeUInt32LE(fileSize, 2); + buffer.writeUInt32LE(54, 10); + + buffer.writeUInt32LE(40, 14); + buffer.writeInt32LE(width, 18); + buffer.writeInt32LE(height, 22); + buffer.writeUInt16LE(1, 26); + buffer.writeUInt16LE(24, 28); + buffer.writeUInt32LE(0, 30); + buffer.writeUInt32LE(imageSize, 34); + + rows.slice().reverse().forEach(function(row, rowIndex) { + const rowOffset = 54 + (rowIndex * rowStride); + const rowBuffer = buffer.subarray(rowOffset, rowOffset + rowStride); + + row.forEach(function(hex, x) { + writeColor(rowBuffer, rowStride, 3, x, 0, hex); + }); + }); + + return buffer; +} + +module.exports = { + create24BitBMP: create24BitBMP, + makeBitmap: makeBitmap +}; diff --git a/test/image.js b/test/image.js index 13af5104..9f596392 100644 --- a/test/image.js +++ b/test/image.js @@ -2,72 +2,7 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); const robot = require('..'); - -function writeColor(buffer, byteWidth, bytesPerPixel, x, y, hex) { - const offset = (y * byteWidth) + (x * bytesPerPixel); - const color = parseInt(hex, 16); - - buffer[offset] = color & 0xFF; - buffer[offset + 1] = (color >> 8) & 0xFF; - buffer[offset + 2] = (color >> 16) & 0xFF; - if (bytesPerPixel === 4) { - buffer[offset + 3] = 0xFF; - } -} - -function makeBitmap(rows, options) { - options = options || {}; - - const width = rows[0].length; - const height = rows.length; - const bytesPerPixel = options.bytesPerPixel || 4; - const byteWidth = options.byteWidth || (width * bytesPerPixel); - const image = Buffer.alloc(byteWidth * height, 0x00); - - rows.forEach(function(row, y) { - row.forEach(function(hex, x) { - writeColor(image, byteWidth, bytesPerPixel, x, y, hex); - }); - }); - - return new robot.Image(width, height, byteWidth, bytesPerPixel * 8, bytesPerPixel, image); -} - -function create24BitBMP(rows) { - const width = rows[0].length; - const height = rows.length; - const rowStride = (width * 3 + 3) & ~3; - const imageSize = rowStride * height; - const fileSize = 54 + imageSize; - const buffer = Buffer.alloc(fileSize, 0x00); - - buffer.writeUInt16LE(0x4D42, 0); - buffer.writeUInt32LE(fileSize, 2); - buffer.writeUInt32LE(54, 10); - - buffer.writeUInt32LE(40, 14); - buffer.writeInt32LE(width, 18); - buffer.writeInt32LE(height, 22); - buffer.writeUInt16LE(1, 26); - buffer.writeUInt16LE(24, 28); - buffer.writeUInt32LE(0, 30); - buffer.writeUInt32LE(imageSize, 34); - - rows.slice().reverse().forEach(function(row, rowIndex) { - const rowOffset = 54 + (rowIndex * rowStride); - - row.forEach(function(hex, x) { - const color = parseInt(hex, 16); - const pixelOffset = rowOffset + (x * 3); - - buffer[pixelOffset] = color & 0xFF; - buffer[pixelOffset + 1] = (color >> 8) & 0xFF; - buffer[pixelOffset + 2] = (color >> 16) & 0xFF; - }); - }); - - return buffer; -} +const { create24BitBMP, makeBitmap } = require('./helpers/images'); describe('Image', function() { it('Loads a BMP file and uses it as a search bitmap.', function() { From bbaa5efe7d213f6318b16405e1ee8f329f9a4cf4 Mon Sep 17 00:00:00 2001 From: Jason Stallings Date: Sat, 14 Mar 2026 12:37:17 -0500 Subject: [PATCH 6/8] fix: Screen captures did not need to be flipped on macOS. --- src/screengrab.c | 4 ---- test/integration/screen.js | 37 ++++++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/screengrab.c b/src/screengrab.c index 7e1ba3f3..96e51d6d 100644 --- a/src/screengrab.c +++ b/src/screengrab.c @@ -75,10 +75,6 @@ MMBitmapRef copyMMBitmapFromDisplayInRect(MMRect rect) return NULL; } - /* RobotJS bitmaps use a top-left origin. Flip the Quartz context so the - * captured image keeps that coordinate system after color conversion. */ - CGContextTranslateCTM(context, 0, height); - CGContextScaleCTM(context, 1, -1); CGContextSetBlendMode(context, kCGBlendModeCopy); CGContextSetInterpolationQuality(context, kCGInterpolationNone); CGContextDrawImage(context, CGRectMake(0, 0, width, height), image); diff --git a/test/integration/screen.js b/test/integration/screen.js index 2b4b0dc3..4575fba0 100644 --- a/test/integration/screen.js +++ b/test/integration/screen.js @@ -27,17 +27,32 @@ describe('Integration/Screen', () => { target = null; }); - it('reads a pixel color', (done) => { - const maxDelay = 1000 - const expected = 'c0ff33' + it('reads a pixel color', () => { const color_1 = elements.color_1; - const sleepTime = robot.getPixelColor(color_1.x, color_1.y) === expected ? 0 - : maxDelay - - setTimeout(() => { - const color = robot.getPixelColor(color_1.x, color_1.y); - expect(color).toEqual(expected); - done() - }, sleepTime) + const color = robot.getPixelColor(color_1.x, color_1.y); + expect(color).toEqual('c0ff33'); + }); + + it('captures the full screen with the same top-left origin as getPixelColor', (done) => { + try { + const capture = robot.screen.capture(); + const expectedTopLeft = { + x: Math.round((elements.color_1.x - 25) * capture.scaleX), + y: Math.round((elements.color_1.y - 25) * capture.scaleY) + }; + const probeCenter = { + x: Math.round(elements.color_1.x * capture.scaleX), + y: Math.round(elements.color_1.y * capture.scaleY) + }; + + expect(robot.getPixelColor(elements.color_1.x, elements.color_1.y)).toEqual('c0ff33'); + expect(capture.findColor('c0ff33')).toEqual(expectedTopLeft); + expect(capture.colorAt(probeCenter.x, probeCenter.y)).toEqual('c0ff33'); + + done(); + } catch (error) { + done.fail(error); + } + }); }); From 2de732bb07324023ef75ce2aeecac40fa0beb5d7 Mon Sep 17 00:00:00 2001 From: Jason Stallings Date: Tue, 17 Mar 2026 12:33:29 -0500 Subject: [PATCH 7/8] fix: Send hotkeys as HIDEvents on macOS. I just wanted to change screens with Control+Right arrow. --- src/keypress.c | 60 ++++++++++++++++++++++++++++++++++++++++++-------- src/robotjs.cc | 10 ++++++--- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/src/keypress.c b/src/keypress.c index c15e31bc..237b156f 100644 --- a/src/keypress.c +++ b/src/keypress.c @@ -48,6 +48,29 @@ static io_connect_t _getAuxiliaryKeyDriver(void) } return sEventDrvrRef; } + +static IOOptionBits nxEventFlagsForMMKeyFlags(MMKeyFlags flags) +{ + IOOptionBits nxFlags = 0; + + if (flags & MOD_META) nxFlags |= NX_COMMANDMASK; + if (flags & MOD_ALT) nxFlags |= NX_ALTERNATEMASK; + if (flags & MOD_CONTROL) nxFlags |= NX_CONTROLMASK; + if (flags & MOD_SHIFT) nxFlags |= NX_SHIFTMASK; + + return nxFlags; +} + +static IOOptionBits nxEventFlagsForModifierKeyCode(MMKeyCode code) +{ + if (code == K_META) return NX_COMMANDMASK; + if (code == K_ALT || code == K_RIGHT_ALT) return NX_ALTERNATEMASK; + if (code == K_CONTROL || code == K_LEFT_CONTROL || code == K_RIGHT_CONTROL) return NX_CONTROLMASK; + if (code == K_SHIFT || code == K_RIGHTSHIFT) return NX_SHIFTMASK; + if (code == K_CAPSLOCK) return NX_ALPHASHIFTMASK; + + return 0; +} #endif #if defined(IS_WINDOWS) @@ -125,14 +148,33 @@ void toggleKeyCode(MMKeyCode code, const bool down, MMKeyFlags flags) kr = IOHIDPostEvent( _getAuxiliaryKeyDriver(), NX_SYSDEFINED, loc, &event, kNXEventDataVersion, 0, FALSE ); assert( KERN_SUCCESS == kr ); } else { - CGEventRef keyEvent = CGEventCreateKeyboardEvent(NULL, - (CGKeyCode)code, down); - assert(keyEvent != NULL); - - CGEventSetType(keyEvent, down ? kCGEventKeyDown : kCGEventKeyUp); - CGEventSetFlags(keyEvent, flags); - CGEventPost(kCGSessionEventTap, keyEvent); - CFRelease(keyEvent); + kern_return_t kr; + IOGPoint loc = { 0, 0 }; + NXEventData event; + IOOptionBits eventFlags; + + bzero(&event, sizeof(NXEventData)); + event.key.repeat = FALSE; + event.key.charCode = 0; + event.key.charSet = 0; + event.key.origCharCode = 0; + event.key.origCharSet = 0; + event.key.keyboardType = 0; + event.key.keyCode = (UInt16)code; + + eventFlags = nxEventFlagsForMMKeyFlags(flags); + if (down) { + eventFlags |= nxEventFlagsForModifierKeyCode(code); + } + + kr = IOHIDPostEvent(_getAuxiliaryKeyDriver(), + down ? NX_KEYDOWN : NX_KEYUP, + loc, + &event, + kNXEventDataVersion, + eventFlags, + kIOHIDSetGlobalEventFlags); + assert(KERN_SUCCESS == kr); } #elif defined(IS_WINDOWS) const DWORD dwFlags = down ? 0 : KEYEVENTF_KEYUP; @@ -241,7 +283,7 @@ void toggleUnicode(UniChar ch, const bool down) CGEventKeyboardSetUnicodeString(keyEvent, 1, (UniChar*) &ch); } - CGEventPost(kCGSessionEventTap, keyEvent); + CGEventPost(kCGHIDEventTap, keyEvent); CFRelease(keyEvent); } #elif defined(USE_X11) diff --git a/src/robotjs.cc b/src/robotjs.cc index 057e0b53..86048b67 100644 --- a/src/robotjs.cc +++ b/src/robotjs.cc @@ -521,6 +521,13 @@ Napi::Value keyTapWrapper(const Napi::CallbackInfo& info) MMKeyFlags flags = MOD_NONE; MMKeyCode key; + + if (info.Length() < 1 || info.Length() > 2) + { + Napi::Error::New(env, "Invalid number of arguments.").ThrowAsJavaScriptException(); +return env.Null(); + } + std::string kstr = info[0].ToString().Utf8Value(); const char *k = kstr.c_str(); @@ -541,9 +548,6 @@ return env.Null(); break; case 1: break; - default: - Napi::Error::New(env, "Invalid number of arguments.").ThrowAsJavaScriptException(); -return env.Null(); } switch(CheckKeyCodes(k, &key)) From 535ba9fab61549335526ac27a50ae46116f85188 Mon Sep 17 00:00:00 2001 From: Jason Stallings Date: Tue, 17 Mar 2026 13:15:42 -0500 Subject: [PATCH 8/8] feat: Add screen.getDisplays() to help with multi-monitor automation. --- index.d.ts | 10 +++ src/mouse.c | 13 +--- src/mouse.h | 2 +- src/robotjs.cc | 157 ++++++++++++++++++++++++++++++++++++++----- src/screen.c | 171 ++++++++++++++++++++++++++++++++++++++++++++++- src/screen.h | 17 +++++ src/screengrab.c | 88 +++++++++++++++++++++--- src/screengrab.h | 6 +- test/screen.js | 17 ++++- 9 files changed, 441 insertions(+), 40 deletions(-) diff --git a/index.d.ts b/index.d.ts index e092e8f6..0c99a318 100644 --- a/index.d.ts +++ b/index.d.ts @@ -10,6 +10,15 @@ export interface ScreenPoint { y: number } +export interface Display { + id: number + x: number + y: number + width: number + height: number + isMain: boolean +} + export interface BitmapSearchOptions { x?: number y?: number @@ -85,6 +94,7 @@ export function scrollMouse(x: number, y: number) : void export function getMousePos(): { x: number, y: number } export function getPixelColor(x: number, y: number): string export function getScreenSize(): { width: number, height: number } +export function getDisplays(): Display[] export var screen: Screen export var image: ImageModule diff --git a/src/mouse.c b/src/mouse.c index b50d7ec0..84cbbef4 100644 --- a/src/mouse.c +++ b/src/mouse.c @@ -371,10 +371,9 @@ static double crude_hypot(double x, double y) return ((M_SQRT2 - 1.0) * small) + big; } -bool smoothlyMoveMouse(MMPoint endPoint,double speed) +bool smoothlyMoveMouse(MMSignedPoint endPoint,double speed) { MMSignedPoint pos = getMousePos(); - MMSize screenSize = getMainDisplaySize(); double velo_x = 0.0, velo_y = 0.0; double distance; @@ -390,14 +389,8 @@ bool smoothlyMoveMouse(MMPoint endPoint,double speed) velo_x /= veloDistance; velo_y /= veloDistance; - pos.x += floor(velo_x + 0.5); - pos.y += floor(velo_y + 0.5); - - /* Make sure we are in the screen boundaries! - * (Strange things will happen if we are not.) */ - if (pos.x >= (int32_t)screenSize.width || pos.y >= (int32_t)screenSize.height) { - return false; - } + pos.x += (int32_t)floor(velo_x + 0.5); + pos.y += (int32_t)floor(velo_y + 0.5); moveMouse(pos); diff --git a/src/mouse.h b/src/mouse.h index a8135ffa..30d5ac75 100644 --- a/src/mouse.h +++ b/src/mouse.h @@ -79,7 +79,7 @@ void dragMouse(MMSignedPoint point, const MMMouseButton button); * * Returns false if unsuccessful (i.e. a point was hit that is outside of the * screen boundaries), or true if successful. */ -bool smoothlyMoveMouse(MMPoint point,double speed); +bool smoothlyMoveMouse(MMSignedPoint point,double speed); /* Returns the coordinates of the mouse on the current screen. */ MMSignedPoint getMousePos(void); diff --git a/src/robotjs.cc b/src/robotjs.cc index 86048b67..13a280af 100644 --- a/src/robotjs.cc +++ b/src/robotjs.cc @@ -141,14 +141,14 @@ Napi::Value moveMouseSmoothWrapper(const Napi::CallbackInfo& info) Napi::Error::New(env, "Invalid number of arguments.").ThrowAsJavaScriptException(); return env.Null(); } - size_t x = static_cast(info[0].ToNumber().Int32Value()); - size_t y = static_cast(info[1].ToNumber().Int32Value()); + int32_t x = info[0].ToNumber().Int32Value(); + int32_t y = info[1].ToNumber().Int32Value(); - MMPoint point; - point = MMPointMake(x, y); + MMSignedPoint point; + point = MMSignedPointMake(x, y); if (info.Length() == 3) { - size_t speed = static_cast(info[2].ToNumber().Int32Value()); + double speed = info[2].ToNumber().DoubleValue(); smoothlyMoveMouse(point, speed); } else @@ -785,6 +785,75 @@ static bool parseSizeT(Napi::Env env, Napi::Value value, const char *name, size_ return true; } +static bool parseInt32(Napi::Env env, Napi::Value value, const char *name, int32_t *result) +{ + if (!value.IsNumber()) { + Napi::Error::New(env, std::string(name) + " must be a number.") + .ThrowAsJavaScriptException(); + return false; + } + + double number = value.As().DoubleValue(); + if (number < static_cast(INT32_MIN) || + number > static_cast(INT32_MAX)) { + Napi::Error::New(env, std::string(name) + " is outside the supported range.") + .ThrowAsJavaScriptException(); + return false; + } + + *result = static_cast(number); + return true; +} + +static bool rectFitsOnDisplay(MMDisplay display, int32_t x, int32_t y, + size_t width, size_t height, MMRect *localRect) +{ + if (width == 0 || height == 0 || + width > static_cast(INT64_MAX) || + height > static_cast(INT64_MAX)) { + return false; + } + + const int64_t left = x; + const int64_t top = y; + const int64_t right = left + static_cast(width); + const int64_t bottom = top + static_cast(height); + const int64_t displayLeft = display.x; + const int64_t displayTop = display.y; + const int64_t displayRight = displayLeft + static_cast(display.width); + const int64_t displayBottom = displayTop + static_cast(display.height); + + if (left < displayLeft || top < displayTop || + right > displayRight || bottom > displayBottom) { + return false; + } + + if (localRect != NULL) { + *localRect = MMRectMake(static_cast(left - displayLeft), + static_cast(top - displayTop), + width, + height); + } + + return true; +} + +static bool findDisplayForRect(const MMDisplay *displayList, size_t displayCount, + int32_t x, int32_t y, size_t width, size_t height, + MMDisplay *display, MMRect *localRect) +{ + for (size_t index = 0; index < displayCount; ++index) { + if (rectFitsOnDisplay(displayList[index], x, y, width, height, localRect)) { + if (display != NULL) { + *display = displayList[index]; + } + return true; + } + } + + return false; +} + static bool parseToleranceOption(Napi::Env env, Napi::Object options, float *tolerance) { if (!options.Has("tolerance")) { @@ -1132,6 +1201,35 @@ Napi::Value getScreenSizeWrapper(const Napi::CallbackInfo& info) return obj; } +Napi::Value getDisplaysWrapper(const Napi::CallbackInfo& info) +{ + Napi::Env env = info.Env(); + size_t displayCount = 0; + MMDisplay *displayList = getDisplayList(&displayCount); + + if (displayList == NULL || displayCount == 0) { + Napi::Error::New(env, "Failed to enumerate active displays.") + .ThrowAsJavaScriptException(); + return env.Null(); + } + + Napi::Array displays = Napi::Array::New(env, displayCount); + + for (size_t index = 0; index < displayCount; ++index) { + Napi::Object display = Napi::Object::New(env); + display.Set("id", Napi::Number::New(env, displayList[index].id)); + display.Set("x", Napi::Number::New(env, displayList[index].x)); + display.Set("y", Napi::Number::New(env, displayList[index].y)); + display.Set("width", Napi::Number::New(env, displayList[index].width)); + display.Set("height", Napi::Number::New(env, displayList[index].height)); + display.Set("isMain", Napi::Boolean::New(env, displayList[index].isMain)); + displays.Set(index, display); + } + + destroyDisplayList(displayList); + return displays; +} + Napi::Value getXDisplayNameWrapper(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); @@ -1162,17 +1260,18 @@ Napi::Value setXDisplayNameWrapper(const Napi::CallbackInfo& info) Napi::Value captureScreenWrapper(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); - size_t x = 0; - size_t y = 0; + int32_t x = 0; + int32_t y = 0; size_t w; size_t h; - MMSize displaySize = getMainDisplaySize(); + bool hasBounds = false; //If user has provided screen coords, use them! if (info.Length() == 4) { - if (!parseSizeT(env, info[0], "x", &x) || - !parseSizeT(env, info[1], "y", &y) || + hasBounds = true; + if (!parseInt32(env, info[0], "x", &x) || + !parseInt32(env, info[1], "y", &y) || !parseSizeT(env, info[2], "width", &w) || !parseSizeT(env, info[3], "height", &h)) { return env.Null(); @@ -1180,6 +1279,7 @@ Napi::Value captureScreenWrapper(const Napi::CallbackInfo& info) } else if (info.Length() == 0) { + MMSize displaySize = getMainDisplaySize(); //We're getting the full screen. w = displaySize.width; h = displaySize.height; @@ -1190,15 +1290,39 @@ Napi::Value captureScreenWrapper(const Napi::CallbackInfo& info) return env.Null(); } - if (w == 0 || h == 0 || - x > displaySize.width || y > displaySize.height || - w > displaySize.width - x || h > displaySize.height - y) { - Napi::Error::New(env, "Requested capture rect is outside the main screen's dimensions.") + if (w == 0 || h == 0) { + Napi::Error::New(env, "Requested capture rect is outside the active displays' dimensions.") .ThrowAsJavaScriptException(); return env.Null(); } - MMBitmapRef bitmap = copyMMBitmapFromDisplayInRect(MMRectMake(x, y, w, h)); + MMBitmapRef bitmap = NULL; + + if (hasBounds) { + size_t displayCount = 0; + MMDisplay *displayList = getDisplayList(&displayCount); + MMDisplay display; + MMRect localRect; + + if (displayList == NULL || displayCount == 0) { + Napi::Error::New(env, "Failed to enumerate active displays.") + .ThrowAsJavaScriptException(); + return env.Null(); + } + + if (!findDisplayForRect(displayList, displayCount, x, y, w, h, &display, &localRect)) { + destroyDisplayList(displayList); + Napi::Error::New(env, "Requested capture rect is outside the active displays' dimensions.") + .ThrowAsJavaScriptException(); + return env.Null(); + } + + bitmap = copyMMBitmapFromDisplayInRectOnDisplay(display, localRect); + destroyDisplayList(displayList); + } else { + bitmap = copyMMBitmapFromDisplayInRect(MMRectMake(0, 0, w, h)); + } + if (bitmap == NULL) { Napi::Error::New(env, "Failed to capture the requested screen rect.") .ThrowAsJavaScriptException(); @@ -1657,6 +1781,9 @@ Napi::Object InitAll(Napi::Env env, Napi::Object exports) exports.Set(Napi::String::New(env, "getScreenSize"), Napi::Function::New(env, getScreenSizeWrapper)); + exports.Set(Napi::String::New(env, "getDisplays"), + Napi::Function::New(env, getDisplaysWrapper)); + exports.Set(Napi::String::New(env, "captureScreen"), Napi::Function::New(env, captureScreenWrapper)); diff --git a/src/screen.c b/src/screen.c index e43b6fda..1f6d3ad0 100644 --- a/src/screen.c +++ b/src/screen.c @@ -1,11 +1,61 @@ #include "screen.h" #include "os.h" +#include #if defined(IS_MACOSX) #include #elif defined(USE_X11) #include #include "xdisplay.h" +#elif defined(IS_WINDOWS) +typedef struct _MMMonitorEnumContext { + MMDisplay *displayList; + size_t count; + size_t capacity; +} MMMonitorEnumContext; + +static BOOL CALLBACK collectMonitor(HMONITOR monitor, HDC hdc, LPRECT rect, LPARAM data) +{ + MMMonitorEnumContext *context = (MMMonitorEnumContext *)data; + MONITORINFO monitorInfo; + MMDisplay *nextList = NULL; + + (void)hdc; + (void)rect; + + if (context == NULL) { + return FALSE; + } + + monitorInfo.cbSize = sizeof(monitorInfo); + if (!GetMonitorInfo(monitor, &monitorInfo)) { + return TRUE; + } + + if (context->count == context->capacity) { + size_t nextCapacity = context->capacity == 0 ? 4 : context->capacity * 2; + nextList = realloc(context->displayList, nextCapacity * sizeof(MMDisplay)); + if (nextList == NULL) { + return FALSE; + } + + context->displayList = nextList; + context->capacity = nextCapacity; + } + + context->displayList[context->count].id = (uint32_t)(context->count + 1); + context->displayList[context->count].x = (int32_t)monitorInfo.rcMonitor.left; + context->displayList[context->count].y = (int32_t)monitorInfo.rcMonitor.top; + context->displayList[context->count].width = + (size_t)(monitorInfo.rcMonitor.right - monitorInfo.rcMonitor.left); + context->displayList[context->count].height = + (size_t)(monitorInfo.rcMonitor.bottom - monitorInfo.rcMonitor.top); + context->displayList[context->count].isMain = + (monitorInfo.dwFlags & MONITORINFOF_PRIMARY) != 0; + ++context->count; + + return TRUE; +} #endif MMSize getMainDisplaySize(void) @@ -16,8 +66,13 @@ MMSize getMainDisplaySize(void) CGDisplayPixelsHigh(displayID)); #elif defined(USE_X11) Display *display = XGetMainDisplay(); - const int screen = DefaultScreen(display); + int screen; + + if (display == NULL) { + return MMSizeMake(0, 0); + } + screen = DefaultScreen(display); return MMSizeMake((size_t)DisplayWidth(display, screen), (size_t)DisplayHeight(display, screen)); #elif defined(IS_WINDOWS) @@ -26,6 +81,120 @@ MMSize getMainDisplaySize(void) #endif } +MMDisplay *getDisplayList(size_t *count) +{ + if (count == NULL) { + return NULL; + } + +#if defined(IS_MACOSX) + uint32_t displayCount = 0; + const uint32_t maxDisplays = 32; + CGDirectDisplayID mainDisplayID = CGMainDisplayID(); + CGDirectDisplayID displayIDs[32]; + MMDisplay *displayList = NULL; + + *count = 0; + + if (CGGetActiveDisplayList(maxDisplays, displayIDs, &displayCount) != kCGErrorSuccess || + displayCount == 0) { + displayCount = 1; + displayList = calloc(displayCount, sizeof(MMDisplay)); + if (displayList == NULL) { + return NULL; + } + + CGRect bounds = CGDisplayBounds(mainDisplayID); + displayList[0].id = mainDisplayID; + displayList[0].x = (int32_t)bounds.origin.x; + displayList[0].y = (int32_t)bounds.origin.y; + displayList[0].width = (size_t)bounds.size.width; + displayList[0].height = (size_t)bounds.size.height; + displayList[0].isMain = true; + *count = 1; + return displayList; + } + + displayList = calloc(displayCount, sizeof(MMDisplay)); + if (displayList == NULL) { + return NULL; + } + + for (uint32_t index = 0; index < displayCount; ++index) { + CGRect bounds = CGDisplayBounds(displayIDs[index]); + displayList[index].id = displayIDs[index]; + displayList[index].x = (int32_t)bounds.origin.x; + displayList[index].y = (int32_t)bounds.origin.y; + displayList[index].width = (size_t)bounds.size.width; + displayList[index].height = (size_t)bounds.size.height; + displayList[index].isMain = displayIDs[index] == mainDisplayID; + } + + *count = displayCount; + return displayList; +#elif defined(USE_X11) + Display *display = XGetMainDisplay(); + MMDisplay *displayList = NULL; + int screen; + + if (display == NULL) { + *count = 0; + return NULL; + } + + screen = DefaultScreen(display); + + displayList = calloc(1, sizeof(MMDisplay)); + if (displayList == NULL) { + *count = 0; + return NULL; + } + + displayList[0].id = (uint32_t)screen; + displayList[0].x = 0; + displayList[0].y = 0; + displayList[0].width = (size_t)DisplayWidth(display, screen); + displayList[0].height = (size_t)DisplayHeight(display, screen); + displayList[0].isMain = true; + *count = 1; + return displayList; +#elif defined(IS_WINDOWS) + MMDisplay *displayList = calloc(1, sizeof(MMDisplay)); + MMMonitorEnumContext context; + + context.displayList = NULL; + context.count = 0; + context.capacity = 0; + + if (EnumDisplayMonitors(NULL, NULL, collectMonitor, (LPARAM)&context)) { + if (context.count > 0) { + *count = context.count; + return context.displayList; + } + } + + free(context.displayList); + if (displayList == NULL) { + *count = 0; + return NULL; + } + + displayList[0].id = 1; + displayList[0].x = 0; + displayList[0].y = 0; + displayList[0].width = (size_t)GetSystemMetrics(SM_CXSCREEN); + displayList[0].height = (size_t)GetSystemMetrics(SM_CYSCREEN); + displayList[0].isMain = true; + *count = 1; + return displayList; +#endif +} + +void destroyDisplayList(MMDisplay *displayList) +{ + free(displayList); +} + bool pointVisibleOnMainDisplay(MMPoint point) { MMSize displaySize = getMainDisplaySize(); diff --git a/src/screen.h b/src/screen.h index 68860ff9..f8804210 100644 --- a/src/screen.h +++ b/src/screen.h @@ -15,9 +15,26 @@ extern "C" { #endif +struct _MMDisplay { + uint32_t id; + int32_t x; + int32_t y; + size_t width; + size_t height; + bool isMain; +}; + +typedef struct _MMDisplay MMDisplay; + /* Returns the size of the main display. */ MMSize getMainDisplaySize(void); +/* Returns the active display list. Caller must free with destroyDisplayList(). */ +MMDisplay *getDisplayList(size_t *count); + +/* Frees a display list returned by getDisplayList(). */ +void destroyDisplayList(MMDisplay *displayList); + /* Convenience function that returns whether the given point is in the bounds * of the main screen. */ bool pointVisibleOnMainDisplay(MMPoint point); diff --git a/src/screengrab.c b/src/screengrab.c index 96e51d6d..93468ea6 100644 --- a/src/screengrab.c +++ b/src/screengrab.c @@ -1,6 +1,7 @@ #include "screengrab.h" #include "bmp_io.h" #include "endian.h" +#include #include /* malloc() */ #if defined(IS_MACOSX) @@ -24,7 +25,33 @@ static void destroyMMBitmapWindowsDIB(char *bitmapBuffer, void *hint) } #endif -MMBitmapRef copyMMBitmapFromDisplayInRect(MMRect rect) +#if !defined(IS_MACOSX) +static bool getDisplayCaptureOrigin(MMDisplay display, MMRect rect, + int32_t *absoluteX, int32_t *absoluteY) +{ + int64_t sourceX; + int64_t sourceY; + + if (absoluteX == NULL || absoluteY == NULL || + rect.origin.x > (size_t)INT32_MAX || rect.origin.y > (size_t)INT32_MAX) { + return false; + } + + sourceX = (int64_t)display.x + (int64_t)rect.origin.x; + sourceY = (int64_t)display.y + (int64_t)rect.origin.y; + + if (sourceX < INT32_MIN || sourceX > INT32_MAX || + sourceY < INT32_MIN || sourceY > INT32_MAX) { + return false; + } + + *absoluteX = (int32_t)sourceX; + *absoluteY = (int32_t)sourceY; + return true; +} +#endif + +MMBitmapRef copyMMBitmapFromDisplayInRectOnDisplay(MMDisplay display, MMRect rect) { #if defined(IS_MACOSX) @@ -37,7 +64,7 @@ MMBitmapRef copyMMBitmapFromDisplayInRect(MMRect rect) const size_t bytesPerPixel = 4; const size_t bytesPerRow = width * bytesPerPixel; - CGDirectDisplayID displayID = CGMainDisplayID(); + CGDirectDisplayID displayID = (CGDirectDisplayID)display.id; CGImageRef image = CGDisplayCreateImageForRect(displayID, CGRectMake(rect.origin.x, @@ -98,16 +125,22 @@ MMBitmapRef copyMMBitmapFromDisplayInRect(MMRect rect) #elif defined(USE_X11) MMBitmapRef bitmap; + int32_t sourceX; + int32_t sourceY; + Display *xDisplay = XGetMainDisplay(); + + if (xDisplay == NULL || + !getDisplayCaptureOrigin(display, rect, &sourceX, &sourceY)) { + return NULL; + } - Display *display = XOpenDisplay(NULL); - XImage *image = XGetImage(display, - XDefaultRootWindow(display), - (int)rect.origin.x, - (int)rect.origin.y, + XImage *image = XGetImage(xDisplay, + XDefaultRootWindow(xDisplay), + sourceX, + sourceY, (unsigned int)rect.size.width, (unsigned int)rect.size.height, AllPlanes, ZPixmap); - XCloseDisplay(display); if (image == NULL) return NULL; bitmap = createMMBitmap((uint8_t *)image->data, @@ -128,6 +161,12 @@ MMBitmapRef copyMMBitmapFromDisplayInRect(MMRect rect) HBITMAP dib; HGDIOBJ previousObject = NULL; BITMAPINFO bi; + int32_t sourceX; + int32_t sourceY; + + if (!getDisplayCaptureOrigin(display, rect, &sourceX, &sourceY)) { + return NULL; + } /* Initialize bitmap info. */ bi.bmiHeader.biSize = sizeof(bi.bmiHeader); @@ -162,8 +201,8 @@ MMBitmapRef copyMMBitmapFromDisplayInRect(MMRect rect) (int)rect.size.width, (int)rect.size.height, screen, - rect.origin.x, - rect.origin.y, + sourceX, + sourceY, SRCCOPY)) { /* Error copying data. */ @@ -195,3 +234,32 @@ MMBitmapRef copyMMBitmapFromDisplayInRect(MMRect rect) return bitmap; #endif } + +MMBitmapRef copyMMBitmapFromDisplayInRect(MMRect rect) +{ +#if defined(IS_MACOSX) + MMDisplay display; + CGDirectDisplayID displayID = CGMainDisplayID(); + + display.id = displayID; + display.x = 0; + display.y = 0; + display.width = (size_t)CGDisplayPixelsWide(displayID); + display.height = (size_t)CGDisplayPixelsHigh(displayID); + display.isMain = true; + + return copyMMBitmapFromDisplayInRectOnDisplay(display, rect); +#else + MMDisplay display; + MMSize displaySize = getMainDisplaySize(); + + display.id = 0; + display.x = 0; + display.y = 0; + display.width = displaySize.width; + display.height = displaySize.height; + display.isMain = true; + + return copyMMBitmapFromDisplayInRectOnDisplay(display, rect); +#endif +} diff --git a/src/screengrab.h b/src/screengrab.h index f10be42d..1d8c2895 100644 --- a/src/screengrab.h +++ b/src/screengrab.h @@ -2,7 +2,7 @@ #ifndef SCREENGRAB_H #define SCREENGRAB_H -#include "types.h" +#include "screen.h" #include "MMBitmap.h" #ifdef __cplusplus @@ -14,6 +14,10 @@ extern "C" * caller), or NULL on error. */ MMBitmapRef copyMMBitmapFromDisplayInRect(MMRect rect); +/* Returns a raw bitmap of screengrab of the given display in display-local + * coordinates (to be destroyed()'d by caller), or NULL on error. */ +MMBitmapRef copyMMBitmapFromDisplayInRectOnDisplay(MMDisplay display, MMRect rect); + #ifdef __cplusplus } #endif diff --git a/test/screen.js b/test/screen.js index c16e33b6..77d4e13a 100644 --- a/test/screen.js +++ b/test/screen.js @@ -1,5 +1,5 @@ var robot = require('..'); -var pixelColor, screenSize; +var displays, pixelColor, screenSize; describe('Screen', () => { it('Get pixel color.', function() @@ -17,7 +17,7 @@ describe('Screen', () => { expect(function() { robot.getPixelColor(-1, -1); - }).toThrowError(/outside the main screen/); + }).toThrowError(/must be non-negative/); expect(function() { @@ -36,4 +36,17 @@ describe('Screen', () => { expect(screenSize.width !== undefined).toBeTruthy(); expect(screenSize.height !== undefined).toBeTruthy(); }); + + it('Get displays.', function() + { + expect(displays = robot.getDisplays()).toBeTruthy(); + expect(Array.isArray(displays)).toBeTruthy(); + expect(displays.length > 0).toBeTruthy(); + expect(displays[0].id !== undefined).toBeTruthy(); + expect(displays[0].x !== undefined).toBeTruthy(); + expect(displays[0].y !== undefined).toBeTruthy(); + expect(displays[0].width !== undefined).toBeTruthy(); + expect(displays[0].height !== undefined).toBeTruthy(); + expect(displays[0].isMain !== undefined).toBeTruthy(); + }); });