Monday, June 25, 2018

[PCIe] How to read/write PCIe Switch Configuration Space?

Here is a question, how to read/write PCIe Switch Configuration Space? We can see this picture first.





The memory map shows the entire physical address space of the root complex.  Only the green block at the bottom is system DRAM. Those yellow areas above are memory mapped peripherals, including PCIe Switch. So CPU can read PCIe Switch configuration space via MMIO in Host Memory. So Basic Address Registers (BAR) are very important and My laptop doesn't have PCIe switch device so that I just pick up a SATA device and it is a very simple example to read 256 bytes of configuration space as follows:

P.S: If you have PCIe switch device ID, just replace it to the code.

aboutpci.c
/***
reference: http://telesky.pixnet.net/blog/post/7022197-a-simple-linux-driver-example-on-fpga%3A-adder
***/

#include <linux/init.h>
#include <linux/module.h>
#include <linux/pci.h>

MODULE_LICENSE("Dual BSD/GPL");

#define    OUR_SATA_VENDOR_ID    0x14e4
#define    OUR_SATA_PROD_ID    0x165f


void print_addr_func(u8 *src, int size) {
    int i;
    if (size < 0) {
        printk(KERN_ALERT "The size should be greater than 0!\n");
        return;
    }
    
    for(i = 0; i < size; i++) {
        if (! (i & 15))
            printk(KERN_ALERT " %02x:", i);
        printk(KERN_ALERT " %02x", src[i]);
        if ((i & 15) == 15)
            printk(KERN_ALERT "\n");
    }
}


static int aboutpci_init(void)
{
    u8 config_arr[256];
    //int iobase;
    //int iobase_end;
    int i;
    //u8 data_byte = 0;
    //u32 pio_start, pio_end, pio_flags, pio_len = 0;
    unsigned long mmio_start, mmio_end, mmio_flags, mmio_len, ioaddr;
    //u16 data_one_word;
    unsigned int *base_addr, *base_addr_0;

    struct pci_dev *pdev = NULL;

    //Finding the device by Vendor/Device ID Pair
    pdev = pci_get_device(OUR_SATA_VENDOR_ID, OUR_SATA_PROD_ID, pdev);
    if (pdev != NULL) {
        printk(KERN_ALERT "Our SATA HBA found!\n");
        if ( pdev->dma_mask == DMA_BIT_MASK(64) )
            printk(KERN_ALERT "64-bit addressing capable!\n");
        else if ( pdev->dma_mask == DMA_BIT_MASK(32) )
            printk(KERN_ALERT "32-bit addressing capable!\n");
        /* Bus-specific parameters. For a PCI NIC, it looks as follows */
        printk(KERN_ALERT "Use pci_read_config_byte() to print bytes in configuration space\n");
        for(i = 0; i < 256; i++) {
            pci_read_config_byte(pdev, i, &config_arr[i]);
            //printk(KERN_ALERT " %02X ", config_arr[i]);
        }
        print_addr_func(config_arr, 256);

        printk(KERN_ALERT "Use pci_resource_XXX() to access BAR 0\n");
        mmio_start = pci_resource_start (pdev, 0);
        mmio_end = pci_resource_end (pdev, 0);
        mmio_flags = pci_resource_flags (pdev, 0);
        mmio_len = pci_resource_len (pdev, 0);
        
        printk(KERN_ALERT "MMIO region size of BAR 1 is :%lu\n", mmio_len);
 printk(KERN_ALERT "MMIO region base addr is %x\n", mmio_start);

        /* make sure PCI base addr 1 is MMIO */
 if (!(mmio_flags & IORESOURCE_MEM)) {
     printk(KERN_ALERT, "region #1 not an MMIO resource, aborting\n");
 }
        
        // Get BAR0's address
        /* ioremap MMIO region */
 ioaddr = ioremap(mmio_start, mmio_len);
 if (ioaddr == NULL) {
     printk(KERN_ALERT "MMIO region is rrror!! \n");
 }   
 printk(KERN_ALERT "MMIO Remap addr is %x\n", ioaddr);
        // print out the MMIO region content from remap addr (virtual address)
        print_addr_func(ioaddr, 16 /* part of mmio_len */);
    }
    else
        printk(KERN_ALERT "Our SATA HBA Not found!\n");

    //Finding the device by its class code
    pdev = NULL;
    pdev = pci_get_class(PCI_CLASS_STORAGE_SATA_AHCI, pdev);
    if (pdev != NULL) {
        printk(KERN_ALERT "SATA HBA Class device found!\n");
        printk(KERN_ALERT "Device Vendor ID: 0x%X\n", pdev->vendor);
        printk(KERN_ALERT "Device Product ID: 0x%X\n", pdev->device);
     
       /* Bus-specific parameters. For a PCI NIC, it looks as follows */
       //iobase = pci_resource_start(dev, 1);
       //iobase_end = iobase + pci_resource_len(dev, 1);
       //printk(KERN_ALERT "Device class bar0 from: 0x%X to 0x%X\n", iobase, iobase_end);
    }
    else
        printk(KERN_ALERT "SATA HBA Class device Not found!\n");

    return 0;
}

static void aboutpci_exit(void)
{
    printk(KERN_ALERT "Goodbye, pci hackers\n");
}

module_init(aboutpci_init);
module_exit(aboutpci_exit);


Use this Makefile to build your module:
Makefile
ifneq ($(KERNELRELEASE),)
    obj-m := aboutpci.o
else
    KERNELDIR := /lib/modules/$(shell uname -r)/build
    PWD := $(shell pwd)
default:
        $(MAKE) -C $(KERNELDIR) M=$(PWD) modules
endif


Once you have done, you will get the following result in files:
$ ls -al
-rw-rw-r-- 1 liudanny liudanny  3798 Aug 16  2017 aboutpci.c
-rw-rw-r-- 1 liudanny liudanny  6464 Jun 25 11:10 aboutpci.ko
-rw-rw-r-- 1 liudanny liudanny   363 Jun 25 11:10 .aboutpci.ko.cmd
-rw-rw-r-- 1 liudanny liudanny   542 Jun 25 11:10 aboutpci.mod.c
-rw-rw-r-- 1 liudanny liudanny  2536 Jun 25 11:10 aboutpci.mod.o
-rw-rw-r-- 1 liudanny liudanny 28760 Jun 25 11:10 .aboutpci.mod.o.cmd
-rw-rw-r-- 1 liudanny liudanny  5784 Jun 25 11:10 aboutpci.o
-rw-rw-r-- 1 liudanny liudanny 42695 Jun 25 11:10 .aboutpci.o.cmd
-rw-rw-r-- 1 liudanny liudanny   191 Jul 28  2017 Makefile
-rw-rw-r-- 1 liudanny liudanny    77 Jun 25 11:10 modules.order
-rw-rw-r-- 1 liudanny liudanny     0 Jul 28  2017 Module.symvers

Now, you can insert the module and see the kernel message
$ sudo insmod aboutpci.ko
$ dmesg
[338360.940926] 64-bit addressing capable!
[338360.940928] Use pci_read_config_byte() to print bytes in configuration space
[338360.941232]  00:
[338360.941235]  e4 14
[338360.941238]  5f 16
[338360.941240]  06 04
[338360.941242]  10 00
[338360.941245]  00 00
[338360.941247]  00 02
[338360.941249]  08 00
[338360.941251]  80 00
[338360.941253]
[338360.941255]  10:
[338360.941256]  0c 00
[338360.941258]  b3 91
[338360.941260]  00 00
[338360.941263]  00 00
[338360.941265]  0c 00
[338360.941267]  b4 91
[338360.941269]  00 00
[338360.941271]  00 00
[338360.941273]
[338360.941274]  20:
[338360.941276]  0c 00
[338360.941278]  b5 91
[338360.941280]  00 00
[338360.941282]  00 00
[338360.941284]  00 00
[338360.941286]  00 00
[338360.941288]  28 10
[338360.941290]  5b 1f
[338360.941292]
[338360.941294]  30:
[338360.941295]  00 00
[338360.941297]  fc ff
[338360.941299]  48 00
[338360.941301]  00 00
[338360.941303]  00 00
[338360.941305]  00 00
[338360.941307]  0f 01
[338360.941309]  00 00
[338360.941312]
[338360.941313]  40:
[338360.941314]  00 00
[338360.941316]  00 00
[338360.941318]  00 00
[338360.941320]  00 cb
[338360.941322]  01 50
[338360.941324]  03 c8
[338360.941326]  08 20
[338360.941329]  00 64
[338360.941331]
[338360.941332]  50:
[338360.941333]  03 58
[338360.941335]  8c 81
[338360.941337]  00 00
[338360.941340]  00 78
[338360.941342]  05 a0
[338360.941344]  86 00
[338360.941346]  00 00
[338360.941348]  00 00
[338360.941350]
[338360.941351]  60:
[338360.941352]  00 00
[338360.941355]  00 00
[338360.941357]  00 00
[338360.941359]  00 00
[338360.941361]  98 02
[338360.941363]  00 f0
[338360.941365]  81 00
[338360.941367]  38 00
[338360.941369]
[338360.941371]  70:
[338360.941372]  b0 10
[338360.941374]  07 00
[338360.941376]  d8 b6
[338360.941378]  98 f4
[338360.941380]  00 00
[338360.941382]  00 00
[338360.941385]  00 00
[338360.941387]  00 00
[338360.941389]
[338360.941390]  80:
[338360.941391]  e4 14
[338360.941393]  5f 16
[338360.941395]  40 00
[338360.941397]  00 40
[338360.941400]  00 00
[338360.941402]  00 00
[338360.941404]  2e 0e
[338360.941406]  00 00
[338360.941408]
[338360.941409]  90:
[338360.941410]  00 00
[338360.941412]  00 00
[338360.941414]  00 00
[338360.941416]  00 00
[338360.941419]  00 00
[338360.941421]  00 00
[338360.941423]  02 02
[338360.941425]  00 00
[338360.941427]
[338360.941429]  a0:
[338360.941430]  11 ac
[338360.941432]  10 80
[338360.941434]  04 00
[338360.941436]  00 00
[338360.941438]  04 10
[338360.941440]  00 00
[338360.941443]  10 00
[338360.941445]  02 00
[338360.941447]
[338360.941448]  b0:
[338360.941449]  81 8d
[338360.941451]  00 10
[338360.941453]  2e 54
[338360.941456]  10 00
[338360.941458]  22 cc
[338360.941460]  04 00
[338360.941462]  40 00
[338360.941464]  12 10
[338360.941466]
[338360.941468]  c0:
[338360.941469]  00 00
[338360.941471]  00 00
[338360.941473]  00 00
[338360.941475]  00 00
[338360.941477]  00 00
[338360.941479]  00 00
[338360.941481]  00 00
[338360.941483]  00 00
[338360.941485]
[338360.941487]  d0:
[338360.941488]  1f 00
[338360.941490]  00 00
[338360.941492]  06 00
[338360.941494]  00 00
[338360.941496]  00 00
[338360.941498]  00 00
[338360.941500]  01 00
[338360.941502]  00 00
[338360.941504]
[338360.941506]  e0:
[338360.941507]  00 00
[338360.941509]  00 00
[338360.941511]  00 00
[338360.941513]  00 00
[338360.941515]  00 00
[338360.941517]  00 00
[338360.941519]  00 00
[338360.941521]  00 00
[338360.941523]
[338360.941525]  f0:
[338360.941526]  00 00
[338360.941528]  00 00
[338360.941530]  00 00
[338360.941532]  72 05
[338360.941534]  00 00
[338360.941536]  00 00
[338360.941538]  0a 60
[338360.941541]  b0 78
[338360.941543]
[338360.941544] Use pci_resource_XXX() to access BAR 0
[338360.941546] MMIO region size of BAR 1 is :65536
[338360.941548] MMIO region base addr is 91b30000
[338360.941577] MMIO Remap addr is 9240000
[338360.941578]  00:
[338360.941581]  e4 14
[338360.941585]  5f 16
[338360.941589]  06 04
[338360.941593]  10 00
[338360.941597]  00 00
[338360.941601]  00 02
[338360.941605]  08 00
[338360.941610]  80 00
...

More example:
lspci -vvvx | grep Ethernet -a1 -b15
17656:01:00.0 Ethernet controller: Broadcom Corporation NetXtreme BCM5720 Gigabit Ethernet PCIe
17746-  Subsystem: Dell Device 1f5b
17775-  Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx+
17878-  Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
17980-  Latency: 0, Cache Line Size: 32 bytes
18019-  Interrupt: pin A routed to IRQ 82
18054-  Region 0: Memory at 91b30000 (64-bit, prefetchable) [size=64K]
18118-  Region 2: Memory at 91b40000 (64-bit, prefetchable) [size=64K]
18182-  Region 4: Memory at 91b50000 (64-bit, prefetchable) [size=64K]
18246-  Expansion ROM at 92000000 [disabled] [size=256K]
18296-  Capabilities: <access denied>
18327-  Kernel driver in use: tg3
18354-00: e4 14 5f 16 06 04 10 00 00 00 00 02 08 00 80 00
18406-10: 0c 00 b3 91 00 00 00 00 0c 00 b4 91 00 00 00 00
18458-20: 0c 00 b5 91 00 00 00 00 00 00 00 00 28 10 5b 1f
18510-30: 00 00 fc ff 48 00 00 00 00 00 00 00 0f 01 00 00


[338360.940926] 64-bit addressing capable!
[338360.940928] Use pci_read_config_byte() to print bytes in configuration space
[338360.941232]  00:
[338360.941235]  e4 14
[338360.941238]  5f 16
[338360.941240]  06 04
[338360.941242]  10 00
[338360.941245]  00 00
[338360.941247]  00 02
[338360.941249]  08 00
[338360.941251]  80 00
[338360.941253]
[338360.941255]  10:
[338360.941256]  0c 00
[338360.941258]  b3 91
[338360.941260]  00 00
[338360.941263]  00 00
[338360.941265]  0c 00
[338360.941267]  b4 91
[338360.941269]  00 00
[338360.941271]  00 00
[338360.941273]
[338360.941274]  20:
[338360.941276]  0c 00
[338360.941278]  b5 91
[338360.941280]  00 00
[338360.941282]  00 00
[338360.941284]  00 00
[338360.941286]  00 00
[338360.941288]  28 10
[338360.941290]  5b 1f
[338360.941292]
[338360.941294]  30:
[338360.941295]  00 00
[338360.941297]  fc ff
[338360.941299]  48 00
[338360.941301]  00 00
[338360.941303]  00 00
[338360.941305]  00 00
[338360.941307]  0f 01
[338360.941309]  00 00
[338360.941312]
[338360.941313]  40:
[338360.941314]  00 00
[338360.941316]  00 00
[338360.941318]  00 00
[338360.941320]  00 cb
[338360.941322]  01 50
[338360.941324]  03 c8
[338360.941326]  08 20
[338360.941329]  00 64
[338360.941331]
[338360.941332]  50:
[338360.941333]  03 58
[338360.941335]  8c 81
[338360.941337]  00 00
[338360.941340]  00 78
[338360.941342]  05 a0
[338360.941344]  86 00
[338360.941346]  00 00
[338360.941348]  00 00
[338360.941350]
[338360.941351]  60:
[338360.941352]  00 00
[338360.941355]  00 00
[338360.941357]  00 00
[338360.941359]  00 00
[338360.941361]  98 02
[338360.941363]  00 f0
[338360.941365]  81 00
[338360.941367]  38 00
[338360.941369]
[338360.941371]  70:
[338360.941372]  b0 10
[338360.941374]  07 00
[338360.941376]  d8 b6
[338360.941378]  98 f4
[338360.941380]  00 00
[338360.941382]  00 00
[338360.941385]  00 00
[338360.941387]  00 00
[338360.941389]
[338360.941390]  80:
[338360.941391]  e4 14
[338360.941393]  5f 16
[338360.941395]  40 00
[338360.941397]  00 40
[338360.941400]  00 00
[338360.941402]  00 00
[338360.941404]  2e 0e
[338360.941406]  00 00
[338360.941408]
[338360.941409]  90:
[338360.941410]  00 00
[338360.941412]  00 00
[338360.941414]  00 00
[338360.941416]  00 00
[338360.941419]  00 00
[338360.941421]  00 00
[338360.941423]  02 02
[338360.941425]  00 00
[338360.941427]
[338360.941429]  a0:
[338360.941430]  11 ac
[338360.941432]  10 80
[338360.941434]  04 00
[338360.941436]  00 00
[338360.941438]  04 10
[338360.941440]  00 00
[338360.941443]  10 00
[338360.941445]  02 00
[338360.941447]
[338360.941448]  b0:
[338360.941449]  81 8d
[338360.941451]  00 10
[338360.941453]  2e 54
[338360.941456]  10 00
[338360.941458]  22 cc
[338360.941460]  04 00
[338360.941462]  40 00
[338360.941464]  12 10
[338360.941466]
[338360.941468]  c0:
[338360.941469]  00 00
[338360.941471]  00 00
[338360.941473]  00 00
[338360.941475]  00 00
[338360.941477]  00 00
[338360.941479]  00 00
[338360.941481]  00 00
[338360.941483]  00 00
[338360.941485]
[338360.941487]  d0:
[338360.941488]  1f 00
[338360.941490]  00 00
[338360.941492]  06 00
[338360.941494]  00 00
[338360.941496]  00 00
[338360.941498]  00 00
[338360.941500]  01 00
[338360.941502]  00 00
[338360.941504]
[338360.941506]  e0:
[338360.941507]  00 00
[338360.941509]  00 00
[338360.941511]  00 00
[338360.941513]  00 00
[338360.941515]  00 00
[338360.941517]  00 00
[338360.941519]  00 00
[338360.941521]  00 00
[338360.941523]
[338360.941525]  f0:
[338360.941526]  00 00
[338360.941528]  00 00
[338360.941530]  00 00
[338360.941532]  72 05
[338360.941534]  00 00
[338360.941536]  00 00
[338360.941538]  0a 60
[338360.941541]  b0 78
[338360.941543]
[338360.941544] Use pci_resource_XXX() to access BAR 0
[338360.941546] MMIO region size of BAR 1 is :65536
[338360.941548] MMIO region base addr is 91b30000
[338360.941577] MMIO Remap addr is 9240000
[338360.941578]  00:
[338360.941581]  e4 14
[338360.941585]  5f 16
[338360.941589]  06 04
[338360.941593]  10 00
[338360.941597]  00 00
[338360.941601]  00 02
[338360.941605]  08 00
[338360.941610]  80 00
...

No comments: