formatting and linting automation

Squashed commits:
[cd479ba] formatting and linting automation
[7fbfabb] formatting and linting automation
[c4f9505] fix after rebase
[f506ad4] rename job
[441d517] update doc (+7 squashed commits)
[2e1b416] fix invoke and github action
[6ae5bb4] doc for git blame
[44b5461] add GitHub action
[b07474f] add docs
[4cd9a6f] more linter fixes
[db71901] wip
[540dc88] wip
This commit is contained in:
Gilles Boccon-Gibod
2022-12-10 09:29:51 -08:00
parent 80fe2ea422
commit c2959dadb4
140 changed files with 2632 additions and 1346 deletions

View File

@@ -35,61 +35,76 @@ async def open_transport(name):
Where <parameters> depend on the type (and may be empty for some types).
The supported types are: serial,udp,tcp,pty,usb
'''
# pylint: disable=import-outside-toplevel
# pylint: disable=too-many-return-statements
scheme, *spec = name.split(':', 1)
if scheme == 'serial' and spec:
from .serial import open_serial_transport
return await open_serial_transport(spec[0])
elif scheme == 'udp' and spec:
if scheme == 'udp' and spec:
from .udp import open_udp_transport
return await open_udp_transport(spec[0])
elif scheme == 'tcp-client' and spec:
if scheme == 'tcp-client' and spec:
from .tcp_client import open_tcp_client_transport
return await open_tcp_client_transport(spec[0])
elif scheme == 'tcp-server' and spec:
if scheme == 'tcp-server' and spec:
from .tcp_server import open_tcp_server_transport
return await open_tcp_server_transport(spec[0])
elif scheme == 'ws-client' and spec:
if scheme == 'ws-client' and spec:
from .ws_client import open_ws_client_transport
return await open_ws_client_transport(spec[0])
elif scheme == 'ws-server' and spec:
if scheme == 'ws-server' and spec:
from .ws_server import open_ws_server_transport
return await open_ws_server_transport(spec[0])
elif scheme == 'pty':
if scheme == 'pty':
from .pty import open_pty_transport
return await open_pty_transport(spec[0] if spec else None)
elif scheme == 'file':
if scheme == 'file':
from .file import open_file_transport
return await open_file_transport(spec[0] if spec else None)
elif scheme == 'vhci':
if scheme == 'vhci':
from .vhci import open_vhci_transport
return await open_vhci_transport(spec[0] if spec else None)
elif scheme == 'hci-socket':
if scheme == 'hci-socket':
from .hci_socket import open_hci_socket_transport
return await open_hci_socket_transport(spec[0] if spec else None)
elif scheme == 'usb':
if scheme == 'usb':
from .usb import open_usb_transport
return await open_usb_transport(spec[0] if spec else None)
elif scheme == 'pyusb':
if scheme == 'pyusb':
from .pyusb import open_pyusb_transport
return await open_pyusb_transport(spec[0] if spec else None)
elif scheme == 'android-emulator':
if scheme == 'android-emulator':
from .android_emulator import open_android_emulator_transport
return await open_android_emulator_transport(spec[0] if spec else None)
else:
raise ValueError('unknown transport scheme')
raise ValueError('unknown transport scheme')
# -----------------------------------------------------------------------------
@@ -104,5 +119,5 @@ async def open_transport_or_link(name):
link.close()
return LinkTransport(controller, AsyncPipeSink(controller))
else:
return await open_transport(name)
return await open_transport(name)

View File

@@ -65,9 +65,12 @@ class PacketPump:
# -----------------------------------------------------------------------------
class PacketParser:
'''
In-line parser that accepts data and emits 'on_packet' when a full packet has been parsed
In-line parser that accepts data and emits 'on_packet' when a full packet has been
parsed
'''
# pylint: disable=attribute-defined-outside-init
NEED_TYPE = 0
NEED_LENGTH = 1
NEED_BODY = 2
@@ -278,7 +281,7 @@ class PumpedPacketSource(ParserSource):
logger.debug('source pump task done')
break
except Exception as error:
logger.warn(f'exception while waiting for packet: {error}')
logger.warning(f'exception while waiting for packet: {error}')
self.terminated.set_result(error)
break
@@ -309,7 +312,7 @@ class PumpedPacketSink:
logger.debug('sink pump task done')
break
except Exception as error:
logger.warn(f'exception while sending packet: {error}')
logger.warning(f'exception while sending packet: {error}')
break
self.pump_task = asyncio.create_task(pump_packets())

View File

@@ -30,8 +30,9 @@ logger = logging.getLogger(__name__)
# -----------------------------------------------------------------------------
async def open_file_transport(spec):
'''
Open a File transport (typically not for a real file, but for a PTY or other unix virtual files).
The parameter string is the path of the file to open
Open a File transport (typically not for a real file, but for a PTY or other unix
virtual files).
The parameter string is the path of the file to open.
'''
# Open the file
@@ -39,12 +40,12 @@ async def open_file_transport(spec):
# Setup reading
read_transport, packet_source = await asyncio.get_running_loop().connect_read_pipe(
lambda: StreamPacketSource(), file
StreamPacketSource, file
)
# Setup writing
write_transport, _ = await asyncio.get_running_loop().connect_write_pipe(
lambda: asyncio.BaseProtocol(), file
asyncio.BaseProtocol, file
)
packet_sink = StreamPacketSink(write_transport)

View File

@@ -40,7 +40,7 @@ async def open_hci_socket_transport(spec):
or a 0-based integer to indicate the adapter number.
'''
HCI_CHANNEL_USER = 1
HCI_CHANNEL_USER = 1 # pylint: disable=invalid-name
# Create a raw HCI socket
try:
@@ -49,10 +49,12 @@ async def open_hci_socket_transport(spec):
socket.SOCK_RAW | socket.SOCK_NONBLOCK,
socket.BTPROTO_HCI,
)
except AttributeError:
except AttributeError as error:
# Not supported on this platform
logger.info("HCI sockets not supported on this platform")
raise Exception('Bluetooth HCI sockets not supported on this platform')
raise Exception(
'Bluetooth HCI sockets not supported on this platform'
) from error
# Compute the adapter index
if spec is None:
@@ -66,13 +68,19 @@ async def open_hci_socket_transport(spec):
try:
ctypes.cdll.LoadLibrary('libc.so.6')
libc = ctypes.CDLL('libc.so.6', use_errno=True)
except OSError:
except OSError as error:
logger.info("HCI sockets not supported on this platform")
raise Exception('Bluetooth HCI sockets not supported on this platform')
raise Exception(
'Bluetooth HCI sockets not supported on this platform'
) from error
libc.bind.argtypes = (ctypes.c_int, ctypes.POINTER(ctypes.c_char), ctypes.c_int)
libc.bind.restype = ctypes.c_int
bind_address = struct.pack(
'<HHH', socket.AF_BLUETOOTH, adapter_index, HCI_CHANNEL_USER
# pylint: disable=no-member
'<HHH',
socket.AF_BLUETOOTH,
adapter_index,
HCI_CHANNEL_USER,
)
if (
libc.bind(
@@ -85,9 +93,9 @@ async def open_hci_socket_transport(spec):
raise IOError(ctypes.get_errno(), os.strerror(ctypes.get_errno()))
class HciSocketSource(ParserSource):
def __init__(self, socket):
def __init__(self, hci_socket):
super().__init__()
self.socket = socket
self.socket = hci_socket
asyncio.get_running_loop().add_reader(
socket.fileno(), self.recv_until_would_block
)
@@ -107,8 +115,8 @@ async def open_hci_socket_transport(spec):
asyncio.get_running_loop().remove_reader(self.socket.fileno())
class HciSocketSink:
def __init__(self, socket):
self.socket = socket
def __init__(self, hci_socket):
self.socket = hci_socket
self.packets = collections.deque()
self.writer_added = False
@@ -127,10 +135,13 @@ async def open_hci_socket_transport(spec):
break
if self.packets:
# There's still something to send, ensure that we are monitoring the socket
# There's still something to send, ensure that we are monitoring the
# socket
if not self.writer_added:
asyncio.get_running_loop().add_writer(
socket.fileno(), self.send_until_would_block
# pylint: disable=no-member
socket.fileno(),
self.send_until_would_block,
)
self.writer_added = True
else:
@@ -148,9 +159,9 @@ async def open_hci_socket_transport(spec):
asyncio.get_running_loop().remove_writer(self.socket.fileno())
class HciSocketTransport(Transport):
def __init__(self, socket, source, sink):
def __init__(self, hci_socket, source, sink):
super().__init__(source, sink)
self.socket = socket
self.socket = hci_socket
async def close(self):
logger.debug('closing HCI socket transport')

View File

@@ -47,11 +47,11 @@ async def open_pty_transport(spec):
tty.setraw(replica)
read_transport, packet_source = await asyncio.get_running_loop().connect_read_pipe(
lambda: StreamPacketSource(), io.open(primary, 'rb', closefd=False)
StreamPacketSource, io.open(primary, 'rb', closefd=False)
)
write_transport, _ = await asyncio.get_running_loop().connect_write_pipe(
lambda: asyncio.BaseProtocol(), io.open(primary, 'wb', closefd=False)
asyncio.BaseProtocol, io.open(primary, 'wb', closefd=False)
)
packet_sink = StreamPacketSink(write_transport)

View File

@@ -17,11 +17,12 @@
# -----------------------------------------------------------------------------
import asyncio
import logging
import threading
import time
import libusb_package
import usb.core
import usb.util
import threading
import time
from colors import color
from .common import Transport, ParserSource
@@ -49,6 +50,7 @@ async def open_pyusb_transport(spec):
04b4:f901 --> the BT USB dongle with vendor=04b4 and product=f901
'''
# pylint: disable=invalid-name
USB_RECIPIENT_DEVICE = 0x00
USB_REQUEST_TYPE_CLASS = 0x01 << 5
USB_ENDPOINT_EVENTS_IN = 0x81
@@ -109,7 +111,7 @@ async def open_pyusb_transport(spec):
def run(self):
while self.stop_event is None:
time.sleep(1)
self.loop.call_soon_threadsafe(lambda: self.stop_event.set())
self.loop.call_soon_threadsafe(self.stop_event.set)
class UsbPacketSource(asyncio.Protocol, ParserSource):
def __init__(self, device, sco_enabled):
@@ -117,6 +119,7 @@ async def open_pyusb_transport(spec):
self.device = device
self.loop = asyncio.get_running_loop()
self.queue = asyncio.Queue()
self.dequeue_task = None
self.event_thread = threading.Thread(
target=self.run, args=(USB_ENDPOINT_EVENTS_IN, hci.HCI_EVENT_PACKET)
)
@@ -135,8 +138,8 @@ async def open_pyusb_transport(spec):
)
self.sco_thread.stop_event = None
def data_received(self, packet):
self.parser.feed_data(packet)
def data_received(self, data):
self.parser.feed_data(data)
def enqueue(self, packet):
self.queue.put_nowait(packet)
@@ -180,16 +183,17 @@ async def open_pyusb_transport(spec):
except usb.core.USBTimeoutError:
continue
except usb.core.USBError:
# Don't log this: because pyusb doesn't really support multiple threads
# reading at the same time, we can get occasional USBError(errno=5)
# Input/Output errors reported, but they seem to be harmless.
# Don't log this: because pyusb doesn't really support multiple
# threads reading at the same time, we can get occasional
# USBError(errno=5) Input/Output errors reported, but they seem to
# be harmless.
# Until support for async or multi-thread support is added to pyusb,
# we'll just live with this as is...
# logger.warning(f'USB read error: {error}')
time.sleep(1) # Sleep one second to avoid busy looping
stop_event = current_thread.stop_event
self.loop.call_soon_threadsafe(lambda: stop_event.set())
self.loop.call_soon_threadsafe(stop_event.set)
class UsbTransport(Transport):
def __init__(self, device, source, sink):
@@ -243,6 +247,7 @@ async def open_pyusb_transport(spec):
# Select an alternate setting for SCO, if available
sco_enabled = False
# pylint: disable=line-too-long
# NOTE: this is disabled for now, because SCO with alternate settings is broken,
# see: https://github.com/libusb/libusb/issues/36
#

View File

@@ -60,7 +60,7 @@ async def open_serial_transport(spec):
device = spec
serial_transport, packet_source = await serial_asyncio.create_serial_connection(
asyncio.get_running_loop(),
lambda: StreamPacketSource(),
StreamPacketSource,
device,
baudrate=speed,
rtscts=rtscts,

View File

@@ -37,13 +37,13 @@ async def open_tcp_client_transport(spec):
'''
class TcpPacketSource(StreamPacketSource):
def connection_lost(self, error):
logger.debug(f'connection lost: {error}')
self.terminated.set_result(error)
def connection_lost(self, exc):
logger.debug(f'connection lost: {exc}')
self.terminated.set_result(exc)
remote_host, remote_port = spec.split(':')
tcp_transport, packet_source = await asyncio.get_running_loop().create_connection(
lambda: TcpPacketSource(),
TcpPacketSource,
host=remote_host,
port=int(remote_port),
)

View File

@@ -49,8 +49,8 @@ async def open_tcp_server_transport(spec):
# Called when a new connection is established
def connection_made(self, transport):
peername = transport.get_extra_info('peername')
logger.debug('connection from {}'.format(peername))
peer_name = transport.get_extra_info('peer_name')
logger.debug(f'connection from {peer_name}')
self.packet_sink.transport = transport
# Called when the client is disconnected

View File

@@ -57,7 +57,7 @@ async def open_udp_transport(spec):
udp_transport,
packet_source,
) = await asyncio.get_running_loop().create_datagram_endpoint(
lambda: UdpPacketSource(),
UdpPacketSource,
local_addr=(local_host, int(local_port)),
remote_addr=(remote_host, int(remote_port)),
)

View File

@@ -17,12 +17,13 @@
# -----------------------------------------------------------------------------
import asyncio
import logging
import libusb_package
import usb1
import threading
import collections
import ctypes
import platform
import libusb_package
import usb1
from colors import color
from .common import Transport, ParserSource
@@ -39,9 +40,9 @@ logger = logging.getLogger(__name__)
def load_libusb():
'''
Attempt to load the libusb-1.0 C library from libusb_package in site-packages.
If library exists, we create a DLL object and initialize the usb1 backend.
This only needs to be done once, but bufore a usb1.USBContext is created.
If library does not exists, do nothing and usb1 will search default system paths
If the library exists, we create a DLL object and initialize the usb1 backend.
This only needs to be done once, but before a usb1.USBContext is created.
If the library does not exists, do nothing and usb1 will search default system paths
when usb1.USBContext is created.
'''
if libusb_path := libusb_package.get_library_path():
@@ -49,6 +50,7 @@ def load_libusb():
libusb_dll = dll_loader(libusb_path, use_errno=True, use_last_error=True)
usb1.loadLibrary(libusb_dll)
async def open_usb_transport(spec):
'''
Open a USB transport.
@@ -60,21 +62,26 @@ async def open_usb_transport(spec):
With <index> as the 0-based index to select amongst all the devices that appear
to be supporting Bluetooth HCI (0 being the first one), or
Where <vendor> and <product> are the vendor ID and product ID in hexadecimal. The
/<serial-number> suffix or #<index> suffix max be specified when more than one device with
the same vendor and product identifiers are present.
/<serial-number> suffix or #<index> suffix max be specified when more than one
device with the same vendor and product identifiers are present.
In addition, if the moniker ends with the symbol "!", the device will be used in "forced" mode:
the first USB interface of the device will be used, regardless of the interface class/subclass.
This may be useful for some devices that use a custom class/subclass but may nonetheless work as-is.
In addition, if the moniker ends with the symbol "!", the device will be used in
"forced" mode:
the first USB interface of the device will be used, regardless of the interface
class/subclass.
This may be useful for some devices that use a custom class/subclass but may
nonetheless work as-is.
Examples:
0 --> the first BT USB dongle
04b4:f901 --> the BT USB dongle with vendor=04b4 and product=f901
04b4:f901#2 --> the third USB device with vendor=04b4 and product=f901
04b4:f901/00E04C239987 --> the BT USB dongle with vendor=04b4 and product=f901 and serial number 00E04C239987
04b4:f901/00E04C239987 --> the BT USB dongle with vendor=04b4 and product=f901 and
serial number 00E04C239987
usb:0B05:17CB! --> the BT USB dongle vendor=0B05 and product=17CB, in "forced" mode.
'''
# pylint: disable=invalid-name
USB_RECIPIENT_DEVICE = 0x00
USB_REQUEST_TYPE_CLASS = 0x01 << 5
USB_DEVICE_CLASS_DEVICE = 0x00
@@ -125,6 +132,7 @@ async def open_usb_transport(spec):
status = transfer.getStatus()
# logger.debug(f'<<< USB out transfer callback: status={status}')
# pylint: disable=no-member
if status == usb1.TRANSFER_COMPLETED:
self.loop.call_soon_threadsafe(self.on_packet_sent_)
elif status == usb1.TRANSFER_CANCELLED:
@@ -165,15 +173,20 @@ async def open_usb_transport(spec):
else:
logger.warning(color(f'unsupported packet type {packet_type}', 'red'))
async def close(self):
def close(self):
self.closed = True
async def terminate(self):
if not self.closed:
self.close()
# Empty the packet queue so that we don't send any more data
self.packets.clear()
# If we have a transfer in flight, cancel it
if self.transfer.isSubmitted():
# Try to cancel the transfer, but that may fail because it may have already completed
# Try to cancel the transfer, but that may fail because it may have
# already completed
try:
self.transfer.cancel()
@@ -192,12 +205,15 @@ async def open_usb_transport(spec):
self.events_in = events_in
self.loop = asyncio.get_running_loop()
self.queue = asyncio.Queue()
self.dequeue_task = None
self.closed = False
self.event_loop_done = self.loop.create_future()
self.cancel_done = {
hci.HCI_EVENT_PACKET: self.loop.create_future(),
hci.HCI_ACL_DATA_PACKET: self.loop.create_future(),
}
self.events_in_transfer = None
self.acl_in_transfer = None
# Create a thread to process events
self.event_thread = threading.Thread(target=self.run)
@@ -228,8 +244,13 @@ async def open_usb_transport(spec):
def on_packet_received(self, transfer):
packet_type = transfer.getUserData()
status = transfer.getStatus()
# logger.debug(f'<<< USB IN transfer callback: status={status} packet_type={packet_type} length={transfer.getActualLength()}')
# logger.debug(
# f'<<< USB IN transfer callback: status={status} '
# f'packet_type={packet_type} '
# f'length={transfer.getActualLength()}'
# )
# pylint: disable=no-member
if status == usb1.TRANSFER_COMPLETED:
packet = (
bytes([packet_type])
@@ -263,6 +284,7 @@ async def open_usb_transport(spec):
self.events_in_transfer.isSubmitted()
or self.acl_in_transfer.isSubmitted()
):
# pylint: disable=no-member
try:
self.context.handleEvents()
except usb1.USBErrorInterrupted:
@@ -271,19 +293,26 @@ async def open_usb_transport(spec):
logger.debug('USB event loop done')
self.loop.call_soon_threadsafe(self.event_loop_done.set_result, None)
async def close(self):
def close(self):
self.closed = True
async def terminate(self):
if not self.closed:
self.close()
self.dequeue_task.cancel()
# Cancel the transfers
for transfer in (self.events_in_transfer, self.acl_in_transfer):
if transfer.isSubmitted():
# Try to cancel the transfer, but that may fail because it may have already completed
# Try to cancel the transfer, but that may fail because it may have
# already completed
packet_type = transfer.getUserData()
try:
transfer.cancel()
logger.debug(
f'waiting for IN[{packet_type}] transfer cancellation to be done...'
f'waiting for IN[{packet_type}] transfer cancellation '
'to be done...'
)
await self.cancel_done[packet_type]
logger.debug(f'IN[{packet_type}] transfer cancellation done')
@@ -314,8 +343,10 @@ async def open_usb_transport(spec):
sink.start()
async def close(self):
await self.source.close()
await self.sink.close()
self.source.close()
self.sink.close()
await self.source.terminate()
await self.sink.terminate()
self.device.releaseInterface(self.interface)
self.device.close()
self.context.close()
@@ -400,6 +431,7 @@ async def open_usb_transport(spec):
# Look for the first interface with the right class and endpoints
def find_endpoints(device):
# pylint: disable-next=too-many-nested-blocks
for (configuration_index, configuration) in enumerate(device):
interface = None
for interface in configuration:
@@ -448,10 +480,13 @@ async def open_usb_transport(spec):
acl_out,
events_in,
)
else:
logger.debug(
f'skipping configuration {configuration_index + 1} / interface {setting.getNumber()}'
)
logger.debug(
f'skipping configuration {configuration_index + 1} / '
f'interface {setting.getNumber()}'
)
return None
endpoints = find_endpoints(found)
if endpoints is None:
@@ -469,6 +504,7 @@ async def open_usb_transport(spec):
device = found.open()
# Auto-detach the kernel driver if supported
# pylint: disable=no-member
if usb1.hasCapability(usb1.CAP_SUPPORTS_DETACH_KERNEL_DRIVER):
try:
logger.debug('auto-detaching kernel driver')

View File

@@ -44,11 +44,13 @@ async def open_ws_server_transport(spec):
source = ParserSource()
sink = PumpedPacketSink(self.send_packet)
self.connection = asyncio.get_running_loop().create_future()
self.server = None
super().__init__(source, sink)
async def serve(self, local_host, local_port):
self.sink.start()
# pylint: disable-next=no-member
self.server = await websockets.serve(
ws_handler=self.on_connection,
host=local_host if local_host != '_' else None,
@@ -58,15 +60,17 @@ async def open_ws_server_transport(spec):
async def on_connection(self, connection):
logger.debug(
f'new connection on {connection.local_address} from {connection.remote_address}'
f'new connection on {connection.local_address} '
f'from {connection.remote_address}'
)
self.connection.set_result(connection)
# pylint: disable=no-member
try:
async for packet in connection:
if type(packet) is bytes:
if isinstance(packet, bytes):
self.source.parser.feed_data(packet)
else:
logger.warn('discarding packet: not a BINARY frame')
logger.warning('discarding packet: not a BINARY frame')
except websockets.WebSocketException as error:
logger.debug(f'exception while receiving packet: {error}')