Skip to content

Commit 296ba51

Browse files
CendioNikosamhed
CendioNiko
and
samhed
committed
Add support for VMware cursor encoding
Supports both classic cursor type and alpha cursor type. In classic mode the server can send 'inverted' pixels for the cursor, our code does not support this but handles these pixels as opaque black. Co-authored-by: Samuel Mannehed <[email protected]>
1 parent 9886d59 commit 296ba51

File tree

3 files changed

+285
-0
lines changed

3 files changed

+285
-0
lines changed

‎core/encodings.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ export const encodings = {
2626
pseudoEncodingContinuousUpdates: -313,
2727
pseudoEncodingCompressLevel9: -247,
2828
pseudoEncodingCompressLevel0: -256,
29+
pseudoEncodingVMwareCursor: 0x574d5664
2930
};
3031

3132
export function encodingName(num) {

‎core/rfb.js‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1239,6 +1239,7 @@ export default class RFB extends EventTargetMixin {
12391239
encs.push(encodings.pseudoEncodingContinuousUpdates);
12401240

12411241
if (this._fb_depth == 24) {
1242+
encs.push(encodings.pseudoEncodingVMwareCursor);
12421243
encs.push(encodings.pseudoEncodingCursor);
12431244
}
12441245

@@ -1488,6 +1489,9 @@ export default class RFB extends EventTargetMixin {
14881489
this._FBU.rects = 1; // Will be decreased when we return
14891490
return true;
14901491

1492+
case encodings.pseudoEncodingVMwareCursor:
1493+
return this._handleVMwareCursor();
1494+
14911495
case encodings.pseudoEncodingCursor:
14921496
return this._handleCursor();
14931497

@@ -1515,6 +1519,122 @@ export default class RFB extends EventTargetMixin {
15151519
}
15161520
}
15171521

1522+
_handleVMwareCursor() {
1523+
const hotx = this._FBU.x; // hotspot-x
1524+
const hoty = this._FBU.y; // hotspot-y
1525+
const w = this._FBU.width;
1526+
const h = this._FBU.height;
1527+
if (this._sock.rQwait("VMware cursor encoding", 1)) {
1528+
return false;
1529+
}
1530+
1531+
const cursor_type = this._sock.rQshift8();
1532+
1533+
this._sock.rQshift8(); //Padding
1534+
1535+
let rgba;
1536+
const bytesPerPixel = 4;
1537+
1538+
//Classic cursor
1539+
if (cursor_type == 0) {
1540+
//Used to filter away unimportant bits.
1541+
//OR is used for correct conversion in js.
1542+
const PIXEL_MASK = 0xffffff00 | 0;
1543+
rgba = new Array(w * h * bytesPerPixel);
1544+
1545+
if (this._sock.rQwait("VMware cursor classic encoding",
1546+
(w * h * bytesPerPixel) * 2, 2)) {
1547+
return false;
1548+
}
1549+
1550+
let and_mask = new Array(w * h);
1551+
for (let pixel = 0; pixel < (w * h); pixel++) {
1552+
and_mask[pixel] = this._sock.rQshift32();
1553+
}
1554+
1555+
let xor_mask = new Array(w * h);
1556+
for (let pixel = 0; pixel < (w * h); pixel++) {
1557+
xor_mask[pixel] = this._sock.rQshift32();
1558+
}
1559+
1560+
for (let pixel = 0; pixel < (w * h); pixel++) {
1561+
if (and_mask[pixel] == 0) {
1562+
//Fully opaque pixel
1563+
let bgr = xor_mask[pixel];
1564+
let r = bgr >> 8 & 0xff;
1565+
let g = bgr >> 16 & 0xff;
1566+
let b = bgr >> 24 & 0xff;
1567+
1568+
rgba[(pixel * bytesPerPixel) ] = r; //r
1569+
rgba[(pixel * bytesPerPixel) + 1 ] = g; //g
1570+
rgba[(pixel * bytesPerPixel) + 2 ] = b; //b
1571+
rgba[(pixel * bytesPerPixel) + 3 ] = 0xff; //a
1572+
1573+
} else if ((and_mask[pixel] & PIXEL_MASK) ==
1574+
PIXEL_MASK) {
1575+
//Only screen value matters, no mouse colouring
1576+
if (xor_mask[pixel] == 0) {
1577+
//Transparent pixel
1578+
rgba[(pixel * bytesPerPixel) ] = 0x00;
1579+
rgba[(pixel * bytesPerPixel) + 1 ] = 0x00;
1580+
rgba[(pixel * bytesPerPixel) + 2 ] = 0x00;
1581+
rgba[(pixel * bytesPerPixel) + 3 ] = 0x00;
1582+
1583+
} else if ((xor_mask[pixel] & PIXEL_MASK) ==
1584+
PIXEL_MASK) {
1585+
//Inverted pixel, not supported in browsers.
1586+
//Fully opaque instead.
1587+
rgba[(pixel * bytesPerPixel) ] = 0x00;
1588+
rgba[(pixel * bytesPerPixel) + 1 ] = 0x00;
1589+
rgba[(pixel * bytesPerPixel) + 2 ] = 0x00;
1590+
rgba[(pixel * bytesPerPixel) + 3 ] = 0xff;
1591+
1592+
} else {
1593+
//Unhandled xor_mask
1594+
rgba[(pixel * bytesPerPixel) ] = 0x00;
1595+
rgba[(pixel * bytesPerPixel) + 1 ] = 0x00;
1596+
rgba[(pixel * bytesPerPixel) + 2 ] = 0x00;
1597+
rgba[(pixel * bytesPerPixel) + 3 ] = 0xff;
1598+
}
1599+
1600+
} else {
1601+
//Unhandled and_mask
1602+
rgba[(pixel * bytesPerPixel) ] = 0x00;
1603+
rgba[(pixel * bytesPerPixel) + 1 ] = 0x00;
1604+
rgba[(pixel * bytesPerPixel) + 2 ] = 0x00;
1605+
rgba[(pixel * bytesPerPixel) + 3 ] = 0xff;
1606+
}
1607+
}
1608+
1609+
//Alpha cursor.
1610+
} else if (cursor_type == 1) {
1611+
if (this._sock.rQwait("VMware cursor alpha encoding",
1612+
(w * h * 4), 2)) {
1613+
return false;
1614+
}
1615+
1616+
rgba = new Array(w * h * bytesPerPixel);
1617+
1618+
for (let pixel = 0; pixel < (w * h); pixel++) {
1619+
let data = this._sock.rQshift32();
1620+
1621+
rgba[(pixel * 4) ] = data >> 8 & 0xff; //r
1622+
rgba[(pixel * 4) + 1 ] = data >> 16 & 0xff; //g
1623+
rgba[(pixel * 4) + 2 ] = data >> 24 & 0xff; //b
1624+
rgba[(pixel * 4) + 3 ] = data & 0xff; //a
1625+
}
1626+
1627+
} else {
1628+
Log.Warn("The given cursor type is not supported: "
1629+
+ cursor_type + " given.");
1630+
return false;
1631+
}
1632+
1633+
this._updateCursor(rgba, hotx, hoty, w, h);
1634+
1635+
return true;
1636+
}
1637+
15181638
_handleCursor() {
15191639
const hotx = this._FBU.x; // hotspot-x
15201640
const hoty = this._FBU.y; // hotspot-y

‎tests/test.rfb.js‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2124,6 +2124,170 @@ describe('Remote Frame Buffer Protocol Client', function () {
21242124
});
21252125
});
21262126

2127+
describe('the VMware Cursor pseudo-encoding handler', function () {
2128+
beforeEach(function () {
2129+
sinon.spy(client._cursor, 'change');
2130+
});
2131+
afterEach(function () {
2132+
client._cursor.change.resetHistory();
2133+
});
2134+
2135+
it('should handle the VMware cursor pseudo-encoding', function () {
2136+
let data = [0x00, 0x00, 0xff, 0,
2137+
0x00, 0xff, 0x00, 0,
2138+
0x00, 0xff, 0x00, 0,
2139+
0x00, 0x00, 0xff, 0];
2140+
let rect = [];
2141+
push8(rect, 0);
2142+
push8(rect, 0);
2143+
2144+
//AND-mask
2145+
for (let i = 0; i < data.length; i++) {
2146+
push8(rect, data[i]);
2147+
}
2148+
//XOR-mask
2149+
for (let i = 0; i < data.length; i++) {
2150+
push8(rect, data[i]);
2151+
}
2152+
2153+
send_fbu_msg([{ x: 0, y: 0, width: 2, height: 2,
2154+
encoding: 0x574d5664}],
2155+
[rect], client);
2156+
expect(client._FBU.rects).to.equal(0);
2157+
});
2158+
2159+
it('should handle insufficient cursor pixel data', function () {
2160+
2161+
// Specified 14x23 pixels for the cursor,
2162+
// but only send 2x2 pixels worth of data
2163+
let w = 14;
2164+
let h = 23;
2165+
let data = [0x00, 0x00, 0xff, 0,
2166+
0x00, 0xff, 0x00, 0];
2167+
let rect = [];
2168+
2169+
push8(rect, 0);
2170+
push8(rect, 0);
2171+
2172+
//AND-mask
2173+
for (let i = 0; i < data.length; i++) {
2174+
push8(rect, data[i]);
2175+
}
2176+
//XOR-mask
2177+
for (let i = 0; i < data.length; i++) {
2178+
push8(rect, data[i]);
2179+
}
2180+
2181+
send_fbu_msg([{ x: 0, y: 0, width: w, height: h,
2182+
encoding: 0x574d5664}],
2183+
[rect], client);
2184+
2185+
// expect one FBU to remain unhandled
2186+
expect(client._FBU.rects).to.equal(1);
2187+
});
2188+
2189+
it('should update the cursor when type is classic', function () {
2190+
let and_mask =
2191+
[0xff, 0xff, 0xff, 0xff, //Transparent
2192+
0xff, 0xff, 0xff, 0xff, //Transparent
2193+
0x00, 0x00, 0x00, 0x00, //Opaque
2194+
0xff, 0xff, 0xff, 0xff]; //Inverted
2195+
2196+
let xor_mask =
2197+
[0x00, 0x00, 0x00, 0x00, //Transparent
2198+
0x00, 0x00, 0x00, 0x00, //Transparent
2199+
0x11, 0x22, 0x33, 0x44, //Opaque
2200+
0xff, 0xff, 0xff, 0x44]; //Inverted
2201+
2202+
let rect = [];
2203+
push8(rect, 0); //cursor_type
2204+
push8(rect, 0); //padding
2205+
let hotx = 0;
2206+
let hoty = 0;
2207+
let w = 2;
2208+
let h = 2;
2209+
2210+
//AND-mask
2211+
for (let i = 0; i < and_mask.length; i++) {
2212+
push8(rect, and_mask[i]);
2213+
}
2214+
//XOR-mask
2215+
for (let i = 0; i < xor_mask.length; i++) {
2216+
push8(rect, xor_mask[i]);
2217+
}
2218+
2219+
let expected_rgba = [0x00, 0x00, 0x00, 0x00,
2220+
0x00, 0x00, 0x00, 0x00,
2221+
0x33, 0x22, 0x11, 0xff,
2222+
0x00, 0x00, 0x00, 0xff];
2223+
2224+
send_fbu_msg([{ x: hotx, y: hoty,
2225+
width: w, height: h,
2226+
encoding: 0x574d5664}],
2227+
[rect], client);
2228+
2229+
expect(client._cursor.change)
2230+
.to.have.been.calledOnce;
2231+
expect(client._cursor.change)
2232+
.to.have.been.calledWith(expected_rgba,
2233+
hotx, hoty,
2234+
w, h);
2235+
});
2236+
2237+
it('should update the cursor when type is alpha', function () {
2238+
let data = [0xee, 0x55, 0xff, 0x00, // bgra
2239+
0x00, 0xff, 0x00, 0xff,
2240+
0x00, 0xff, 0x00, 0x22,
2241+
0x00, 0xff, 0x00, 0x22,
2242+
0x00, 0xff, 0x00, 0x22,
2243+
0x00, 0x00, 0xff, 0xee];
2244+
let rect = [];
2245+
push8(rect, 1); //cursor_type
2246+
push8(rect, 0); //padding
2247+
let hotx = 0;
2248+
let hoty = 0;
2249+
let w = 3;
2250+
let h = 2;
2251+
2252+
for (let i = 0; i < data.length; i++) {
2253+
push8(rect, data[i]);
2254+
}
2255+
2256+
let expected_rgba = [0xff, 0x55, 0xee, 0x00,
2257+
0x00, 0xff, 0x00, 0xff,
2258+
0x00, 0xff, 0x00, 0x22,
2259+
0x00, 0xff, 0x00, 0x22,
2260+
0x00, 0xff, 0x00, 0x22,
2261+
0xff, 0x00, 0x00, 0xee];
2262+
2263+
send_fbu_msg([{ x: hotx, y: hoty,
2264+
width: w, height: h,
2265+
encoding: 0x574d5664}],
2266+
[rect], client);
2267+
2268+
expect(client._cursor.change)
2269+
.to.have.been.calledOnce;
2270+
expect(client._cursor.change)
2271+
.to.have.been.calledWith(expected_rgba,
2272+
hotx, hoty,
2273+
w, h);
2274+
});
2275+
2276+
it('should not update cursor when incorrect cursor type given', function () {
2277+
let rect = [];
2278+
push8(rect, 3); // invalid cursor type
2279+
push8(rect, 0); // padding
2280+
2281+
client._cursor.change.resetHistory();
2282+
send_fbu_msg([{ x: 0, y: 0, width: 2, height: 2,
2283+
encoding: 0x574d5664}],
2284+
[rect], client);
2285+
2286+
expect(client._cursor.change)
2287+
.to.not.have.been.called;
2288+
});
2289+
});
2290+
21272291
it('should handle the last_rect pseudo-encoding', function () {
21282292
send_fbu_msg([{ x: 0, y: 0, width: 0, height: 0, encoding: -224}], [[]], client, 100);
21292293
expect(client._FBU.rects).to.equal(0);

0 commit comments

Comments
 (0)