Skip to content

Add support for VMware cursor encoding #1299

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 25, 2019
Merged

Add support for VMware cursor encoding #1299

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions core/encodings.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ export const encodings = {
pseudoEncodingContinuousUpdates: -313,
pseudoEncodingCompressLevel9: -247,
pseudoEncodingCompressLevel0: -256,
pseudoEncodingVMwareCursor: 0x574d5664
};

export function encodingName(num) {
Expand Down
120 changes: 120 additions & 0 deletions core/rfb.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1239,6 +1239,7 @@ export default class RFB extends EventTargetMixin {
encs.push(encodings.pseudoEncodingContinuousUpdates);

if (this._fb_depth == 24) {
encs.push(encodings.pseudoEncodingVMwareCursor);
encs.push(encodings.pseudoEncodingCursor);
}

Expand Down Expand Up @@ -1488,6 +1489,9 @@ export default class RFB extends EventTargetMixin {
this._FBU.rects = 1; // Will be decreased when we return
return true;

case encodings.pseudoEncodingVMwareCursor:
return this._handleVMwareCursor();

case encodings.pseudoEncodingCursor:
return this._handleCursor();

Expand Down Expand Up @@ -1515,6 +1519,122 @@ export default class RFB extends EventTargetMixin {
}
}

_handleVMwareCursor() {
const hotx = this._FBU.x; // hotspot-x
const hoty = this._FBU.y; // hotspot-y
const w = this._FBU.width;
const h = this._FBU.height;
if (this._sock.rQwait("VMware cursor encoding", 1)) {
return false;
}

const cursor_type = this._sock.rQshift8();

this._sock.rQshift8(); //Padding

let rgba;
const bytesPerPixel = 4;

//Classic cursor
if (cursor_type == 0) {
//Used to filter away unimportant bits.
//OR is used for correct conversion in js.
const PIXEL_MASK = 0xffffff00 | 0;
rgba = new Array(w * h * bytesPerPixel);

if (this._sock.rQwait("VMware cursor classic encoding",
(w * h * bytesPerPixel) * 2, 2)) {
return false;
}

let and_mask = new Array(w * h);
for (let pixel = 0; pixel < (w * h); pixel++) {
and_mask[pixel] = this._sock.rQshift32();
}

let xor_mask = new Array(w * h);
for (let pixel = 0; pixel < (w * h); pixel++) {
xor_mask[pixel] = this._sock.rQshift32();
}

for (let pixel = 0; pixel < (w * h); pixel++) {
if (and_mask[pixel] == 0) {
//Fully opaque pixel
let bgr = xor_mask[pixel];
let r = bgr >> 8 & 0xff;
let g = bgr >> 16 & 0xff;
let b = bgr >> 24 & 0xff;

rgba[(pixel * bytesPerPixel) ] = r; //r
rgba[(pixel * bytesPerPixel) + 1 ] = g; //g
rgba[(pixel * bytesPerPixel) + 2 ] = b; //b
rgba[(pixel * bytesPerPixel) + 3 ] = 0xff; //a

} else if ((and_mask[pixel] & PIXEL_MASK) ==
PIXEL_MASK) {
//Only screen value matters, no mouse colouring
if (xor_mask[pixel] == 0) {
//Transparent pixel
rgba[(pixel * bytesPerPixel) ] = 0x00;
rgba[(pixel * bytesPerPixel) + 1 ] = 0x00;
rgba[(pixel * bytesPerPixel) + 2 ] = 0x00;
rgba[(pixel * bytesPerPixel) + 3 ] = 0x00;

} else if ((xor_mask[pixel] & PIXEL_MASK) ==
PIXEL_MASK) {
//Inverted pixel, not supported in browsers.
//Fully opaque instead.
rgba[(pixel * bytesPerPixel) ] = 0x00;
rgba[(pixel * bytesPerPixel) + 1 ] = 0x00;
rgba[(pixel * bytesPerPixel) + 2 ] = 0x00;
rgba[(pixel * bytesPerPixel) + 3 ] = 0xff;

} else {
//Unhandled xor_mask
rgba[(pixel * bytesPerPixel) ] = 0x00;
rgba[(pixel * bytesPerPixel) + 1 ] = 0x00;
rgba[(pixel * bytesPerPixel) + 2 ] = 0x00;
rgba[(pixel * bytesPerPixel) + 3 ] = 0xff;
}

} else {
//Unhandled and_mask
rgba[(pixel * bytesPerPixel) ] = 0x00;
rgba[(pixel * bytesPerPixel) + 1 ] = 0x00;
rgba[(pixel * bytesPerPixel) + 2 ] = 0x00;
rgba[(pixel * bytesPerPixel) + 3 ] = 0xff;
}
}

//Alpha cursor.
} else if (cursor_type == 1) {
if (this._sock.rQwait("VMware cursor alpha encoding",
(w * h * 4), 2)) {
return false;
}

rgba = new Array(w * h * bytesPerPixel);

for (let pixel = 0; pixel < (w * h); pixel++) {
let data = this._sock.rQshift32();

rgba[(pixel * 4) ] = data >> 8 & 0xff; //r
rgba[(pixel * 4) + 1 ] = data >> 16 & 0xff; //g
rgba[(pixel * 4) + 2 ] = data >> 24 & 0xff; //b
rgba[(pixel * 4) + 3 ] = data & 0xff; //a
}

} else {
Log.Warn("The given cursor type is not supported: "
+ cursor_type + " given.");
return false;
}

this._updateCursor(rgba, hotx, hoty, w, h);

return true;
}

_handleCursor() {
const hotx = this._FBU.x; // hotspot-x
const hoty = this._FBU.y; // hotspot-y
Expand Down
164 changes: 164 additions & 0 deletions tests/test.rfb.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2124,6 +2124,170 @@ describe('Remote Frame Buffer Protocol Client', function () {
});
});

describe('the VMware Cursor pseudo-encoding handler', function () {
beforeEach(function () {
sinon.spy(client._cursor, 'change');
});
afterEach(function () {
client._cursor.change.resetHistory();
});

it('should handle the VMware cursor pseudo-encoding', function () {
let data = [0x00, 0x00, 0xff, 0,
0x00, 0xff, 0x00, 0,
0x00, 0xff, 0x00, 0,
0x00, 0x00, 0xff, 0];
let rect = [];
push8(rect, 0);
push8(rect, 0);

//AND-mask
for (let i = 0; i < data.length; i++) {
push8(rect, data[i]);
}
//XOR-mask
for (let i = 0; i < data.length; i++) {
push8(rect, data[i]);
}

send_fbu_msg([{ x: 0, y: 0, width: 2, height: 2,
encoding: 0x574d5664}],
[rect], client);
expect(client._FBU.rects).to.equal(0);
});

it('should handle insufficient cursor pixel data', function () {

// Specified 14x23 pixels for the cursor,
// but only send 2x2 pixels worth of data
let w = 14;
let h = 23;
let data = [0x00, 0x00, 0xff, 0,
0x00, 0xff, 0x00, 0];
let rect = [];

push8(rect, 0);
push8(rect, 0);

//AND-mask
for (let i = 0; i < data.length; i++) {
push8(rect, data[i]);
}
//XOR-mask
for (let i = 0; i < data.length; i++) {
push8(rect, data[i]);
}

send_fbu_msg([{ x: 0, y: 0, width: w, height: h,
encoding: 0x574d5664}],
[rect], client);

// expect one FBU to remain unhandled
expect(client._FBU.rects).to.equal(1);
});

it('should update the cursor when type is classic', function () {
let and_mask =
[0xff, 0xff, 0xff, 0xff, //Transparent
0xff, 0xff, 0xff, 0xff, //Transparent
0x00, 0x00, 0x00, 0x00, //Opaque
0xff, 0xff, 0xff, 0xff]; //Inverted

let xor_mask =
[0x00, 0x00, 0x00, 0x00, //Transparent
0x00, 0x00, 0x00, 0x00, //Transparent
0x11, 0x22, 0x33, 0x44, //Opaque
0xff, 0xff, 0xff, 0x44]; //Inverted

let rect = [];
push8(rect, 0); //cursor_type
push8(rect, 0); //padding
let hotx = 0;
let hoty = 0;
let w = 2;
let h = 2;

//AND-mask
for (let i = 0; i < and_mask.length; i++) {
push8(rect, and_mask[i]);
}
//XOR-mask
for (let i = 0; i < xor_mask.length; i++) {
push8(rect, xor_mask[i]);
}

let expected_rgba = [0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x33, 0x22, 0x11, 0xff,
0x00, 0x00, 0x00, 0xff];

send_fbu_msg([{ x: hotx, y: hoty,
width: w, height: h,
encoding: 0x574d5664}],
[rect], client);

expect(client._cursor.change)
.to.have.been.calledOnce;
expect(client._cursor.change)
.to.have.been.calledWith(expected_rgba,
hotx, hoty,
w, h);
});

it('should update the cursor when type is alpha', function () {
let data = [0xee, 0x55, 0xff, 0x00, // bgra
0x00, 0xff, 0x00, 0xff,
0x00, 0xff, 0x00, 0x22,
0x00, 0xff, 0x00, 0x22,
0x00, 0xff, 0x00, 0x22,
0x00, 0x00, 0xff, 0xee];
let rect = [];
push8(rect, 1); //cursor_type
push8(rect, 0); //padding
let hotx = 0;
let hoty = 0;
let w = 3;
let h = 2;

for (let i = 0; i < data.length; i++) {
push8(rect, data[i]);
}

let expected_rgba = [0xff, 0x55, 0xee, 0x00,
0x00, 0xff, 0x00, 0xff,
0x00, 0xff, 0x00, 0x22,
0x00, 0xff, 0x00, 0x22,
0x00, 0xff, 0x00, 0x22,
0xff, 0x00, 0x00, 0xee];

send_fbu_msg([{ x: hotx, y: hoty,
width: w, height: h,
encoding: 0x574d5664}],
[rect], client);

expect(client._cursor.change)
.to.have.been.calledOnce;
expect(client._cursor.change)
.to.have.been.calledWith(expected_rgba,
hotx, hoty,
w, h);
});

it('should not update cursor when incorrect cursor type given', function () {
let rect = [];
push8(rect, 3); // invalid cursor type
push8(rect, 0); // padding

client._cursor.change.resetHistory();
send_fbu_msg([{ x: 0, y: 0, width: 2, height: 2,
encoding: 0x574d5664}],
[rect], client);

expect(client._cursor.change)
.to.not.have.been.called;
});
});

it('should handle the last_rect pseudo-encoding', function () {
send_fbu_msg([{ x: 0, y: 0, width: 0, height: 0, encoding: -224}], [[]], client, 100);
expect(client._FBU.rects).to.equal(0);
Expand Down