JackrabbitLabs-Releases/CSE-release
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

qemu support code added

Browse Source
This commit is contained in:
Grant Mackey 2024-04-22 19:13:19 -07:00
parent aa92fdbdd1
commit 2b76db29c9
8 changed files with 456 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
fmapi_psc_handler.c
View File

@ -26,12 +26,18 @@
*/
#include <string.h>
#include <pci/pci.h>
/* struct timespec
* timespec_get()
*
*/
#include <time.h>
/* system()
*/
#include <stdlib.h>
/* autl_prnt_buf()
*/
#include <arrayutils.h>
@ -190,11 +196,55 @@ int fmop_psc_cfg(struct mctp *m, struct mctp_action *ma)
rsp.obj.psc_cfg_rsp.data[2] = 0;
rsp.obj.psc_cfg_rsp.data[3] = 0;
if (opts[CLOP_QEMU].set == 1)
{
switch(req.obj.psc_cfg_req.fdbe)
{
case 0x01:
{
__u8 b = pci_read_byte(p->dev, reg);
rsp.obj.psc_cfg_rsp.data[0] = b;
} break;
case 0x03:
{
// Verify word aligned
if ((reg & 0x1) != 0)
goto send;
__u16 w = pci_read_word(p->dev, reg);
rsp.obj.psc_cfg_rsp.data[0] = ( w ) & 0x00FF;
rsp.obj.psc_cfg_rsp.data[1] = ( w >> 8 ) & 0x00FF;
} break;
case 0x0F:
{
// Verify long aligned
if ((reg & 0x3) != 0)
goto send;
__u32 l = pci_read_long(p->dev, reg);
rsp.obj.psc_cfg_rsp.data[0] = ( l ) & 0x00FF;
rsp.obj.psc_cfg_rsp.data[1] = ( l >> 8 ) & 0x00FF;
rsp.obj.psc_cfg_rsp.data[2] = ( l >> 16) & 0x00FF;
rsp.obj.psc_cfg_rsp.data[3] = ( l >> 24) & 0x00FF;
} break;
default:
goto send;
}
}
else
{
if (req.obj.psc_cfg_req.fdbe & 0x01) rsp.obj.psc_cfg_rsp.data[0] = p->cfgspace[reg+0];
if (req.obj.psc_cfg_req.fdbe & 0x02) rsp.obj.psc_cfg_rsp.data[1] = p->cfgspace[reg+1];
if (req.obj.psc_cfg_req.fdbe & 0x04) rsp.obj.psc_cfg_rsp.data[2] = p->cfgspace[reg+2];
if (req.obj.psc_cfg_req.fdbe & 0x08) rsp.obj.psc_cfg_rsp.data[3] = p->cfgspace[reg+3];
}
}
break;
case FMCT_WRITE: // 0x01
@ -204,11 +254,53 @@ int fmop_psc_cfg(struct mctp *m, struct mctp_action *ma)
reg = (req.obj.psc_cfg_req.ext << 8) | req.obj.psc_cfg_req.reg;
if (opts[CLOP_QEMU].set == 1)
{
switch(req.obj.psc_cfg_req.fdbe)
{
case 0x01:
{
pci_write_byte(p->dev, reg, req.obj.psc_cfg_req.data[0]);
} break;
case 0x03:
{
// Verify word aligned
if ((reg & 0x1) != 0)
goto send;
__u16 w = (req.obj.psc_cfg_req.data[1] << 8)
| req.obj.psc_cfg_req.data[0];
pci_write_word(p->dev, reg, w);
} break;
case 0x0F:
{
// Verify long aligned
if ((reg & 0x3) != 0)
goto send;
__u32 l = (req.obj.psc_cfg_req.data[3] << 24)
|(req.obj.psc_cfg_req.data[2] << 16)
|(req.obj.psc_cfg_req.data[1] << 8)
|(req.obj.psc_cfg_req.data[0] );
pci_write_long(p->dev, reg, l);
} break;
default:
goto send;
}
}
else
{
if (req.obj.psc_cfg_req.fdbe & 0x01) p->cfgspace[reg+0] = req.obj.psc_cfg_req.data[0];
if (req.obj.psc_cfg_req.fdbe & 0x02) p->cfgspace[reg+1] = req.obj.psc_cfg_req.data[1];
if (req.obj.psc_cfg_req.fdbe & 0x04) p->cfgspace[reg+2] = req.obj.psc_cfg_req.data[2];
if (req.obj.psc_cfg_req.fdbe & 0x08) p->cfgspace[reg+3] = req.obj.psc_cfg_req.data[3];
}
}
break;
}
@ -349,7 +441,7 @@ int fmop_psc_id(struct mctp *m, struct mctp_action *ma)
}
for ( int i = 0 ; i < cs->num_vcss ; i++ ) {
for ( int j = 0 ; j < MAX_VPPBS_PER_VCS ; j++ ) {
for ( int j = 0 ; j < cs->vcss[i].num ; j++ ) {
if ( cs->vcss[i].vppbs[j].bind_status != FMBS_UNBOUND )
fi->active_vppbs++;
}
@ -623,16 +715,33 @@ int fmop_psc_port_ctrl(struct mctp *m, struct mctp_action *ma)
switch (req.obj.psc_port_ctrl_req.opcode)
{
case FMPO_ASSERT_PERST: // 0x00
{
char cmd[64];
sprintf(cmd, "echo 0 > /sys/bus/pci/slots/%d/power", p->ppid);
IFV(CLVB_ACTIONS) printf("%s ACT: Asserting PERST on PPID: %d\n", now, req.obj.psc_port_ctrl_req.ppid);
// Disable the device
if ( opts[CLOP_QEMU].set == 1 )
rv = system(cmd);
// Set PERST bit
p->perst = 0x1;
break;
} break;
case FMPO_DEASSERT_PERST: // 0x01
{
char cmd[64];
sprintf(cmd, "echo 1 > /sys/bus/pci/slots/%d/power", p->ppid);
IFV(CLVB_ACTIONS) printf("%s ACT: Deasserting PERST on PPID: %d\n", now, req.obj.psc_port_ctrl_req.ppid);
// Enable the device
if ( opts[CLOP_QEMU].set == 1 )
rv = system(cmd);
p->perst = 0x0;
break;
} break;
case FMPO_RESET_PPB: // 0x02
IFV(CLVB_ACTIONS) printf("%s ACT: Resetting PPID: %d\n", now, req.obj.psc_port_ctrl_req.ppid);

34
fmapi_vsc_handler.c
View File

@ -22,6 +22,9 @@
*/
#include <stdio.h>
/* system()
*/
#include <stdlib.h>
/* memset()
*/
#include <string.h>
@ -351,6 +354,21 @@ int fmop_vsc_bind(struct mctp *m, struct mctp_action *ma)
goto send;
}
// Check if this physical port is already bound to a vppb
for ( int i = 0 ; i < cxls->num_vcss ; i++ )
{
struct cxl_vcs *vcs = &cxls->vcss[i];
for ( int k = 0 ; k < vcs->num ; k++ )
{
struct cxl_vppb *vppb = &vcs->vppbs[k];
if ( vppb->ppid == p->ppid )
{
IFV(CLVB_ERRORS) printf("%s ERR: Specified PPID is already bound. PPBID: %d\n", now, req.obj.vsc_bind_req.ppid);
goto send;
}
}
}
STEP // 10: Perform Action
IFV(CLVB_ACTIONS) printf("%s ACT: Binding VCSID: %d vPPBID: %d PPID: %d LDID: 0x%04x\n", now, req.obj.vsc_bind_req.vcsid, req.obj.vsc_bind_req.vppbid, req.obj.vsc_bind_req.ppid, req.obj.vsc_bind_req.ldid);
@ -378,6 +396,14 @@ int fmop_vsc_bind(struct mctp *m, struct mctp_action *ma)
cxls->bos_rc = FMRC_SUCCESS;
cxls->bos_ext = 0;
// If QEMU, enable power to physical device
if ( opts[CLOP_QEMU].set == 1 )
{
char cmd[64];
sprintf(cmd, "echo 1 > /sys/bus/pci/slots/%d/power", p->ppid);
rv = system(cmd);
}
STEP // 11: Prepare Response Object
STEP // 12: Serialize Response Object
@ -704,6 +730,14 @@ int fmop_vsc_unbind(struct mctp *m, struct mctp_action *ma)
cxls->bos_rc = FMRC_SUCCESS;
cxls->bos_ext = 0;
// If QEMU, disable power to physical device
if ( opts[CLOP_QEMU].set == 1 )
{
char cmd[64];
sprintf(cmd, "echo 0 > /sys/bus/pci/slots/%d/power", p->ppid);
rv = system(cmd);
}
STEP // 11: Prepare Response Object
STEP // 12: Serialize Response Object

6
main.c
View File

@ -67,13 +67,15 @@
#define IFV(u) if (opts[CLOP_VERBOSITY].u64 & u)
#define CSLN_PORTS 32
#define CSLN_VCSS 32
#define CSLN_VCSS 16
#define CSLN_VPPBS 256
/* ENUMERATIONS ==============================================================*/
/* STRUCTS ===================================================================*/
struct devices * qemu_devices;
/* PROTOTYPES ================================================================*/
/* GLOBAL VARIABLES ==========================================================*/
@ -136,8 +138,6 @@ int main(int argc, char* argv[])
}
}
// STEP // 4: Build PCI Representation
STEP // 5: Print the state
if (opts[CLOP_PRINT_STATE].set)
cxls_prnt(cxls);

22
options.c
View File

@ -73,7 +73,9 @@ char *STR_CLOP[] = {
"PRINT_STATE",
"PRINT_OPTIONS",
"CONFIG_FILE",
"TCP_PORT"
"TCP_PORT",
"TCP_ADDRESS",
"QEMU"
};
/**
@ -97,7 +99,7 @@ struct opt *opts = NULL;
/**
* Global string used by argp to print version with --version
*/
const char *argp_program_version = "version 0.1";
const char *argp_program_version = "version 0.2";
/**
* Global string used by argp when printing the help message
@ -116,6 +118,8 @@ struct argp_option ao_main[] =
{
{0,0,0,0, "File Options",1},
{"config", 'c', "FILE", 0, "File name of CXL switch config file", 0},
{"qemu-sim", 'q', NULL, OPTION_HIDDEN, "Enable control qemu devices, cse must be run as root", 0}
,
{0,0,0,0, "Networking Options",2},
{"tcp-port", 'P', "INT", 0, "Server TCP Port", 0},
{"tcp-address", 'T', "INT", 0, "Server TCP Address", 0}
@ -193,6 +197,20 @@ static int pr_main (
o->str = strndup(arg, CLMR_MAX_ARG_STR_LEN);
break;
// qemu-sim
case 'q':
if(!getuid())
{
o = &opts[CLOP_QEMU];
o->set = 1;
}
else
{
printf("-q option must be run as root \n");
exit(0);
};
break;
// help
case 'h':
print_help();

1
options.h
View File

@ -89,6 +89,7 @@ enum _CLOP
CLOP_CONFIG_FILE, //!< File to load CXL Switch configuration data <str>
CLOP_TCP_PORT, //!< TCP Port to listen on for connections <u16>
CLOP_TCP_ADDRESS, //!< TCP Address to listen on for connections <u32>
CLOP_QEMU, //!< qemu switches, no emulation (for now)
CLOP_MAX
};

280
state.c
View File

@ -39,6 +39,8 @@ cxl_ *
*/
#include <sys/mman.h>
#include <pci/pci.h>
/** GHashTable
* g_hash_table_foreach()
/ */
@ -96,6 +98,7 @@ int state_load_emulator(struct cxl_switch *state, GHashTable *ht);
int state_load_ports(struct cxl_switch *state, GHashTable *ht);
int state_load_switch(struct cxl_switch *state, GHashTable *ht);
int state_load_vcss(struct cxl_switch *state, GHashTable *ht);
int state_load_from_pci(struct cxl_switch *state);
void _parse_devices(gpointer key, gpointer value, gpointer user_data);
void _parse_device(gpointer key, gpointer value, gpointer user_data);
@ -137,9 +140,10 @@ struct cxl_switch *cxls;
* 3: Parse Emulator configuration
* 4: Parse Devices
* 5: Parse Switch
* 6: Parse Ports
* 7: Parse VCSs
* 8: Free memory allocated for hash table
* 6: Load physical devices if in a QEMU environment
* 7: Parse Ports
* 8: Parse VCSs
* 9: Free memory allocated for hash table
*/
int state_load(struct cxl_switch *state, char *filename)
{
@ -184,17 +188,29 @@ int state_load(struct cxl_switch *state, char *filename)
if (rv != 0)
goto end;
STEP // 6: Parse Ports
STEP // 6: Load physical devices if in a QEMU environment
if (opts[CLOP_QEMU].set == 1)
{
rv = state_load_from_pci(state);
if (rv != 0)
goto end;
goto success;
}
STEP // 7: Parse Ports
rv = state_load_ports(state, ht);
if (rv != 0)
goto end;
STEP // 7: Parse VCSs
STEP // 8: Parse VCSs
rv = state_load_vcss(state, ht);
if (rv != 0)
goto end;
STEP // 8: Free memory allocated for hash table
success:
STEP // 9: Free memory allocated for hash table
yl_free(ht);
rv = 0;
@ -289,6 +305,252 @@ end:
return rv;
}
/**
* Load ports and vcs from physical pci devices
*
* @param ht GHashTable holding contents of config.yaml file
* @return Returns 0 upon success. Non zero otherwise
*
* STEPS
*/
int state_load_from_pci(struct cxl_switch *state)
{
INIT
int rv, cache, mem;
unsigned int num_dvsec, nr;
__u32 l, type, vppbid;
__u16 w;
struct pci_dev *dev, *parent;
struct pci_cap *cap;
struct cxl_port cp;
__u32 fillflags;
ENTER
// Initialize variables
rv = 1;
fillflags = PCI_FILL_IDENT
|PCI_FILL_CLASS
|PCI_FILL_CAPS
|PCI_FILL_EXT_CAPS
|PCI_FILL_PHYS_SLOT
|PCI_FILL_MODULE_ALIAS
|PCI_FILL_LABEL
|PCI_FILL_NUMA_NODE
|PCI_FILL_IO_FLAGS
|PCI_FILL_CLASS_EXT
|PCI_FILL_SUBSYS
|PCI_FILL_PARENT
|PCI_FILL_DRIVER;
STEP // : get pci_access ptr, init pci_access ptr, get all the devices
state->pacc = pci_alloc();
pci_init(state->pacc);
pci_scan_bus(state->pacc);
// STEP 3: Iiterate over all devices
for ( dev = state->pacc->devices ; dev ; dev = dev->next )
{
pci_fill_info(dev, PCI_FILL_CLASS);
// If this is a PCI-to-PCI Bridge then check if it is a CXL upstream port
if ( (dev->device_class >> 8) == 0x06 && (dev->device_class & 0x0FF) == 0x04 )
{
// Get more info about this device
pci_fill_info(dev, fillflags);
// Get PCI Express Capability
cap = pci_find_cap(dev, PCI_CAP_ID_EXP, PCI_CAP_NORMAL);
if (cap == NULL)
continue;
// Determine port type
w = pci_read_word(dev, cap->addr + PCI_EXP_FLAGS);
type = (w & PCI_EXP_FLAGS_TYPE) >> 4;
if (type != PCI_EXP_TYPE_UPSTREAM)
continue;
// Clear the local CXL Port before filling it
memset(&cp, 0, sizeof(cp));
// Get max speed / width / vppbid
l = pci_read_long(dev, cap->addr + PCI_EXP_LNKCAP);
cp.mls = l & PCI_EXP_LNKCAP_SPEED;
cp.mlw = (l & PCI_EXP_LNKCAP_WIDTH) >> 4;
vppbid = l >> 24;
// Get cur speed / width
w = pci_read_word(dev, cap->addr + PCI_EXP_LNKSTA);
cp.cls = w & PCI_EXP_LNKSTA_SPEED;
cp.nlw = (w & PCI_EXP_LNKSTA_WIDTH) >> 4;
// If parent is NULL, then skip this device as we know nothing about it
parent = dev->parent;
if (parent == NULL)
continue;
// Get all info about the parent device
pci_fill_info(parent, fillflags);
// Get PCI Express Capability
cap = pci_find_cap(parent, PCI_CAP_ID_EXP, PCI_CAP_NORMAL);
if (cap == NULL)
continue;
// Get physical port id from Slot Number in PCI Express capability
l = pci_read_long(parent, cap->addr + PCI_EXP_SLTCAP);
cp.ppid = ((l & PCI_EXP_SLTCAP_PSN) >> 19);
// Set switch IDs based on the upstream port identifiers
state->vid = dev->vendor_id;
state->did = dev->device_id;
state->ssid = dev->subsys_id;
state->svid = dev->subsys_vendor_id;
state->sn = ((__u64) dev->domain_16 ) << 48
| ((__u64) dev->device_class ) << 32
| ((__u64) dev->prog_if ) << 24
| ((__u64) dev->bus ) << 16
| ((__u64) dev->dev ) << 8
| ((__u64) dev->func ) ;
// Set bind in vcs
state->vcss[0].uspid = cp.ppid;
state->vcss[0].state = FMVS_ENABLED;
state->vcss[0].vppbs[vppbid].ppid = cp.ppid;
state->vcss[0].vppbs[vppbid].bind_status = FMBS_BOUND_PORT;
state->vcss[0].vppbs[vppbid].ldid = 0;
// Fill local struct cxl_port fields
cp.state = FMPS_USP;
cp.dt = FMDT_CXL_TYPE_1;
cp.speeds = cp.mls;
cp.ltssm = FMLS_L0;
cp.lane = 0;
cp.lane_rev = 0;
cp.perst = 0;
cp.prsnt = 1;
cp.pwrctrl = 0;
cp.dev = dev;
cp.dv = FMDV_CXL2_0;
cp.cv = FMCV_CXL1_1 | FMCV_CXL2_0;
// Copy local struct cxl_port into global state
memcpy(&state->ports[cp.ppid], &cp, sizeof(cp));
}
// If this is a CXL device, then gather more info and the parent's info
else if ( (dev->device_class >> 8) == 0x05 && (dev->device_class & 0x0FF) == 0x02 )
{
// Get more info about this device
pci_fill_info(dev, fillflags);
// If parent is NULL, then skip this device as we know nothing about it
parent = dev->parent;
if (parent == NULL)
continue;
// Get all info about the parent device
pci_fill_info(parent, fillflags);
// Get PCI Express Capability (Parent)
cap = pci_find_cap(parent, PCI_CAP_ID_EXP, PCI_CAP_NORMAL);
if (cap == NULL)
continue;
// Determine port type (Parent)
w = pci_read_word(parent, cap->addr + PCI_EXP_FLAGS);
type = (w & PCI_EXP_FLAGS_TYPE) >> 4;
if ( type != PCI_EXP_TYPE_DOWNSTREAM )
continue;
// Clear the local CXL Port before filling it
memset(&cp, 0, sizeof(cp));
// Get physical port number from SLot number in PCI Express capability (Parent)
l = pci_read_long(parent, cap->addr + PCI_EXP_SLTCAP);
cp.ppid = ((l & PCI_EXP_SLTCAP_PSN) >> 19);
// Get max speed / width / vppbid (Parent)
l = pci_read_long(parent, cap->addr + PCI_EXP_LNKCAP);
cp.mls = l & PCI_EXP_LNKCAP_SPEED;
cp.mlw = (l & PCI_EXP_LNKCAP_WIDTH) >> 4;
vppbid = l >> 24;
// Get cur speeds
w = pci_read_word(parent, cap->addr + PCI_EXP_LNKSTA);
cp.cls = w & PCI_EXP_LNKSTA_SPEED;
cp.nlw = (w & PCI_EXP_LNKSTA_WIDTH) >> 4;
// Get number of DVSEC Capabilities in Dev. Num returned in variable num_dvsec
num_dvsec = 0;
cap = pci_find_cap_nr(dev, PCI_EXT_CAP_ID_DVSEC, PCI_CAP_EXTENDED, &num_dvsec);
// Loop through DVSEC capabilities
for ( nr = 0 ; nr < num_dvsec ; nr++)
{
// Get DVSEC Capability entry number nr
cap = pci_find_cap_nr(dev, PCI_EXT_CAP_ID_DVSEC, PCI_CAP_EXTENDED, &nr);
// Get the DVSEC.type
w = pci_read_long(dev, cap->addr + PCI_DVSEC_HEADER2);
// If DVSEC.type==0, this DVSEC Capability describes device type
if (w == 0)
{
// Get the flags indicating what CXL protocols are supported
w = pci_read_word(dev, cap->addr + PCI_CXL_DEV_CAP);
cache = w & PCI_CXL_DEV_CAP_CACHE;
mem = (w & PCI_CXL_DEV_CAP_MEM) >> 2;
// Determine Device Type
if (cache == 1 && mem == 0) cp.dt = FMDT_CXL_TYPE_1;
else if (cache == 1 && mem == 1) cp.dt = FMDT_CXL_TYPE_2;
else if (cache == 0 && mem == 1) cp.dt = FMDT_CXL_TYPE_3;
break;
}
// If the DVSEC Type is 9 then this is a MLD DVSEC
else if (w == 9)
{
// Set that this is a MLD device
cp.ld = pci_read_word(dev, cap->addr + PCI_CXL_MLD_NUM_LD);
cp.dt = FMDT_CXL_TYPE_3_POOLED;
}
}
// Set bind in vcs
state->vcss[0].state = FMVS_ENABLED;
state->vcss[0].vppbs[vppbid].ppid = cp.ppid;
state->vcss[0].vppbs[vppbid].bind_status = FMBS_BOUND_PORT;
state->vcss[0].vppbs[vppbid].ldid = 0;
// Fill local struct cxl_port fields
cp.state = FMPS_DSP;
cp.speeds = cp.mls;
cp.ltssm = FMLS_L0;
cp.lane = 0;
cp.lane_rev = 0;
cp.perst = 0;
cp.prsnt = 1;
cp.pwrctrl = 0;
cp.dev = dev;
cp.dv = FMDV_CXL2_0;
cp.cv = FMCV_CXL1_1 | FMCV_CXL2_0;
// Copy local struct cxl_port into global state
memcpy(&state->ports[cp.ppid], &cp, sizeof(cp));
}
}
rv = 0;
return rv;
}
/**
* Load port definitions from hash table into memory
*
@ -960,13 +1222,13 @@ void _parse_switch(gpointer key, gpointer value, gpointer user_data)
else if (!strcmp(key, "bos_ext")) s->bos_ext = strtoul(ylo->str, NULL,0);
else if (!strcmp(key, "msg_rsp_limit_n")) s->msg_rsp_limit_n = atoi(ylo->str);
else if (!strcmp(key, "ingress_port")) s->ingress_port = atoi(ylo->str);
else if (!strcmp(key, "num_ports")) s->num_ports = atoi(ylo->str);
else if (!strcmp(key, "num_vcss")) s->num_vcss = atoi(ylo->str);
else if (!strcmp(key, "num_vppbs")) s->num_vppbs = atoi(ylo->str);
else if (!strcmp(key, "num_decoders")) s->num_decoders = atoi(ylo->str);
else if (!strcmp(key, "mlw")) s->mlw = atoi(ylo->str);
else if (!strcmp(key, "speeds")) s->speeds = strtoul(ylo->str, NULL, 0);
else if (!strcmp(key, "mls")) s->mls = atoi(ylo->str);
else if (!strcmp(key, "num_ports")) cxls_init_ports(s, atoi(ylo->str));
else if (!strcmp(key, "num_vcss")) cxls_init_vcss(s, atoi(ylo->str), s->num_vppbs);
else if (!strcmp(key, "num_vppbs")) cxls_init_vcss(s, s->num_vcss, atoi(ylo->str));
rv = 0;