[devices]: Address Dell issue#46 : Adding MUX reset logic to fix prob…

…e failures (#2356)

platform/broadcom/sonic-platform-modules-dell/common/dell_i2c_utils.sh

@@ -18,7 +18,7 @@ i2c_config() {
     done
 
     if [[ "$count" -eq "$MAX_BUS_RETRY" ]]; then
-        echo "ERROR: $@ : i2c bus not created"
+        echo "dell_i2c_utils : ERROR: $@ : i2c bus not created"
         return
     fi
 
@@ -31,7 +31,7 @@ i2c_config() {
     done
 
     if [[ "$count" -eq "$MAX_I2C_OP_RETRY" ]]; then
-        echo "ERROR: $@ : i2c operation failed"
+        echo "dell_i2c_utils : ERROR: $@ : i2c operation failed"
         return
     fi
 }
@@ -53,10 +53,75 @@ i2c_poll_bus_exists() {
     done
 
     if [[ "$count" -eq "$MAX_BUS_RETRY" ]]; then
-        echo "ERROR: $@ : i2c bus not created"
+        echo "dell_i2c_utils : ERROR: $@ : i2c bus not created"
         return 1
     else
         return 0
     fi
 }
 
+# Perform an i2c mux device create
+# Input is of the form:
+# i2c_mux_create mux_driver i2c_addr i2c_bus_num i2c_child_bus_num_start
+# where i2c_bus_num is the bus number in which the mux is to be created and
+# i2c_child_bus_num_start is the first of the 8 bus channels that this mux should create
+i2c_mux_create() {
+    local MAX_MUX_CHANNEL_RETRY=3
+    local MAX_MUX_CHANNELS=8
+    local count=0
+    local i
+    local mux_driver=$1
+    local i2c_addr=$2
+    local i2c_bus_num=$3
+    local i2c_child_bus_num_start=$4
+
+    # Construct the i2c bus, the first and last bus channels that will be created under the MUX
+    i2c_bus=/sys/bus/i2c/devices/i2c-$i2c_bus_num
+    i2c_mux_channel_first=$i2c_bus/i2c-$i2c_child_bus_num_start
+    i2c_mux_channel_last=$i2c_bus/i2c-$(expr $i2c_child_bus_num_start + $MAX_MUX_CHANNELS - 1)
+
+    if  i2c_poll_bus_exists  $i2c_bus; then
+        while [[ "$count" -lt "$MAX_MUX_CHANNEL_RETRY" ]]; do
+            eval "echo $mux_driver $i2c_addr > /sys/bus/i2c/devices/i2c-$i2c_bus_num/new_device" > /dev/null 2>&1
+            ret=$?
+
+            # Give more time for the mux channels to get created based on retries
+            i=0
+            while [[ "$i" -lt "$count" ]]; do
+                sleep 1
+                i=$((i+1))
+            done
+
+            # Check if the (first and last) mux channels got created
+            if [[ $ret -eq "0" && -e $i2c_mux_channel_first && -e $i2c_mux_channel_last ]]; then
+                break;
+            else
+                # If the channel did not get created, remove the mux, reset the mux tree and retry
+                echo "dell_i2c_utils :  ERROR: i2c mux channel not created for $mux_driver,$i2c_addr,$i2c_bus_num"
+                i2c_mux_delete $i2c_addr $i2c_bus_num
+                reset_muxes
+            fi
+
+            count=$((count+1))
+        done
+    fi
+
+    if [[ "$count" -eq "$MAX_MUX_CHANNEL_RETRY" ]]; then
+        echo "dell_i2c_utils : ERROR: $1,$2 : i2c mux channel not created"
+        return
+    fi
+
+    return
+}
+
+# Perform an i2c mux device delete
+# Input is of the form:
+# i2c_mux_delete i2c_addr i2c_bus_num
+i2c_mux_delete() {
+    local i2c_addr
+    local i2c_bus_num
+
+    i2c_addr=$1
+    i2c_bus_num=$2
+    i2c_config "echo $i2c_addr > /sys/bus/i2c/devices/i2c-$i2c_bus_num/delete_device"
+}

platform/broadcom/sonic-platform-modules-dell/debian/platform-modules-s6100.install

@@ -1,7 +1,7 @@
-s6100/scripts/io_rd_wr.py usr/local/bin
 s6100/scripts/iom_power_*.sh usr/local/bin
 s6100/scripts/s6100_platform.sh usr/local/bin
 common/dell_i2c_utils.sh usr/local/bin
+common/io_rd_wr.py usr/local/bin
 common/fstrim.timer etc/systemd/system
 common/fstrim.service etc/systemd/system
 common/platform_reboot usr/share/sonic/device/x86_64-dell_s6100_c2538-r0

platform/broadcom/sonic-platform-modules-dell/debian/platform-modules-z9100.install

@@ -1,6 +1,7 @@
 z9100/scripts/check_qsfp.sh usr/local/bin
 z9100/scripts/z9100_platform.sh usr/local/bin
 common/dell_i2c_utils.sh usr/local/bin
+common/io_rd_wr.py usr/local/bin
 common/fstrim.timer etc/systemd/system
 common/fstrim.service etc/systemd/system
 common/platform_reboot usr/share/sonic/device/x86_64-dell_z9100_c2538-r0

platform/broadcom/sonic-platform-modules-dell/s6100/modules/dell_s6100_iom_cpld.c

@@ -11,6 +11,8 @@
 //iom cpld slave address
 #define  IOM_CPLD_SLAVE_ADD   0x3e
 
+#define  CPLD_SEP_RST0 0x5
+
 //iom cpld ver register
 #define  IOM_CPLD_SLAVE_VER   0x00
 
@@ -384,6 +386,34 @@ static ssize_t set_abs_mask(struct device *dev, struct device_attribute *devattr
     return count;
 }
 
+static ssize_t get_sep_reset(struct device *dev, struct device_attribute *devattr, char *buf)
+{
+    int ret;
+    u8 devdata=0;
+    struct cpld_data *data = dev_get_drvdata(dev);
+
+    ret = dell_s6100_iom_cpld_read(data,IOM_CPLD_SLAVE_ADD,CPLD_SEP_RST0);
+    if(ret < 0)
+        return sprintf(buf, "read error");
+    devdata = (u8)ret & 0xff;
+    return sprintf(buf,"0x%02x\n",devdata);
+}
+
+static ssize_t set_sep_reset(struct device *dev, struct device_attribute *devattr, const char *buf, size_t count)
+{
+    unsigned long devdata;
+    int err;
+    struct cpld_data *data = dev_get_drvdata(dev);
+
+    err = kstrtoul(buf, 16, &devdata);
+    if (err)
+        return err;
+
+    dell_s6100_iom_cpld_write(data,IOM_CPLD_SLAVE_ADD,CPLD_SEP_RST0,(u8)(devdata & 0xff));
+
+    return count;
+}
+
 static DEVICE_ATTR(iom_cpld_vers,S_IRUGO,get_cpldver, NULL);
 static DEVICE_ATTR(qsfp_modprs, S_IRUGO,get_modprs, NULL);
 static DEVICE_ATTR(qsfp_lpmode, S_IRUGO | S_IWUSR,get_lpmode,set_lpmode);
@@ -396,6 +426,7 @@ static DEVICE_ATTR(qsfp_int, S_IRUGO, get_int, NULL);
 static DEVICE_ATTR(qsfp_abs_int, S_IRUGO, get_abs_int, NULL);
 static DEVICE_ATTR(qsfp_int_mask, S_IRUGO | S_IWUSR, get_int_mask, set_int_mask);
 static DEVICE_ATTR(qsfp_abs_mask, S_IRUGO | S_IWUSR, get_abs_mask, set_abs_mask);
+static DEVICE_ATTR(sep_reset, S_IRUGO | S_IWUSR, get_sep_reset, set_sep_reset);
 
 static struct attribute *i2c_cpld_attrs[] = {
     &dev_attr_qsfp_lpmode.attr,
@@ -410,6 +441,7 @@ static struct attribute *i2c_cpld_attrs[] = {
     &dev_attr_qsfp_abs_int.attr,
     &dev_attr_qsfp_int_mask.attr,
     &dev_attr_qsfp_abs_mask.attr,
+    &dev_attr_sep_reset.attr,
     NULL,
 };
 

platform/broadcom/sonic-platform-modules-dell/s6100/scripts/s6100_platform.sh

@@ -21,9 +21,9 @@ init_devnum() {
 # Attach/Detach CPU board mux @ 0x70
 cpu_board_mux() {
     case $1 in
-        "new_device")    i2c_config "echo pca9547 0x70 > /sys/bus/i2c/devices/i2c-${devnum}/$1"
+        "new_device")    i2c_mux_create pca9547 0x70 $devnum 2 
                          ;;
-        "delete_device") i2c_config "echo 0x70 > /sys/bus/i2c/devices/i2c-${devnum}/$1"
+        "delete_device") i2c_mux_delete 0x70 $devnum
                          ;;
         *)               echo "s6100_platform: cpu_board_mux: invalid command !"
                          ;;
@@ -33,9 +33,9 @@ cpu_board_mux() {
 # Attach/Detach Switchboard MUX @ 0x71
 switch_board_mux() {
     case $1 in
-        "new_device")    i2c_config "echo pca9548 0x71 > /sys/bus/i2c/devices/i2c-4/$1"
+        "new_device")    i2c_mux_create pca9548 0x71 4 10
                          ;;
-        "delete_device") i2c_config "echo 0x71 > /sys/bus/i2c/devices/i2c-4/$1"
+        "delete_device") i2c_mux_delete 0x71 4
                          ;;
         *)               echo "s6100_platform: switch_board_mux : invalid command !"
                          ;;
@@ -78,13 +78,17 @@ switch_board_cpld() {
 switch_board_qsfp_mux() {
     case $1 in
         "new_device")
+                      # The mux for the QSFPs spawn {18..25}, {26..33}... {74..81}
+                      # starting at chennel 18 and 16 channels per IOM.
+                      channel_first=18
                       for ((i=9;i>=6;i--));
                       do
                           # 0x71 mux on the IOM 1
                           mux_index=$(expr $i - 5)
                           echo "Attaching PCA9548 $mux_index"
-                          i2c_config "echo pca9548 0x71 > /sys/bus/i2c/devices/i2c-$i/$1"
-                          i2c_config "echo pca9548 0x72 > /sys/bus/i2c/devices/i2c-$i/$1"
+                          i2c_mux_create pca9548 0x71 $i $channel_first
+                          i2c_mux_create pca9548 0x72 $i $(expr $channel_first + 8)
+                          channel_first=$(expr $channel_first + 16)
                       done
                       ;;
         "delete_device")
@@ -93,8 +97,8 @@ switch_board_qsfp_mux() {
                           # 0x71 mux on the IOM 1
                           mux_index=$(expr $i - 5)
                           echo "Detaching PCA9548 $mux_index"
-                          i2c_config "echo 0x71 > /sys/bus/i2c/devices/i2c-$devnum/i2c-$i/$1"
-                          i2c_config "echo 0x72 > /sys/bus/i2c/devices/i2c-$devnum/i2c-$i/$1"
+                          i2c_mux_delete 0x71 $i
+                          i2c_mux_delete 0x72 $i
                       done
                       ;;
         *)            echo "s6100_platform: switch_board_qsfp_mux: invalid command !"
@@ -191,6 +195,28 @@ xcvr_presence_interrupts() {
     esac
 }
 
+# Reset the mux tree
+reset_muxes() {
+    local i
+
+    # Reset the IOM muxes (if they have been already instantiated)
+    for ((i=14;i<=17;i++));
+    do
+        if [[ -e /sys/class/i2c-adapter/i2c-$i/$i-003e ]]; then
+            echo 0xfc > /sys/class/i2c-adapter/i2c-$i/$i-003e/sep_reset
+            echo 0xff > /sys/class/i2c-adapter/i2c-$i/$i-003e/sep_reset
+        fi
+    done
+
+    # Reset the switch card PCA9548A
+    io_rd_wr.py --set --val 0xef --offset 0x110
+    io_rd_wr.py --set --val 0xff --offset 0x110
+
+    # Reset the CPU Card PCA9547
+    io_rd_wr.py --set --val 0xfd --offset 0x20b
+    io_rd_wr.py --set --val 0xff --offset 0x20b
+}
+
 init_devnum
 
 if [[ "$1" == "init" ]]; then

platform/broadcom/sonic-platform-modules-dell/z9100/scripts/z9100_platform.sh

@@ -21,9 +21,9 @@ init_devnum() {
 # Attach/Detach CPU board mux @ 0x70
 cpu_board_mux() {
     case $1 in
-        "new_device")    i2c_config "echo pca9547 0x70 > /sys/bus/i2c/devices/i2c-${devnum}/$1"
+        "new_device")    i2c_mux_create pca9547 0x70 $devnum 2
                          ;;
-        "delete_device") i2c_config "echo 0x70 > /sys/bus/i2c/devices/i2c-${devnum}/$1"
+        "delete_device") i2c_mux_delete 0x70 $devnum
                          ;;
         *)               echo "z9100_platform: cpu_board_mux: invalid command !"
                          ;;
@@ -33,9 +33,9 @@ cpu_board_mux() {
 # Attach/Detach switch board MUX to IOM CPLDs @ 0x71
 switch_board_mux() {
     case $1 in
-        "new_device")    i2c_config "echo pca9548 0x71 > /sys/bus/i2c/devices/i2c-4/$1"
+        "new_device")    i2c_mux_create pca9548 0x71 4 10
                          ;;
-        "delete_device") i2c_config "echo 0x71 > /sys/bus/i2c/devices/i2c-4/$1"
+        "delete_device") i2c_mux_delete 0x71 4
                          ;;
         *)               echo "z9100_platform: switch_board_mux : invalid command !"
                          ;;
@@ -78,12 +78,16 @@ switch_board_cpld() {
 switch_board_qsfp_mux() {
     case $1 in
         "new_device")
+                      # The mux for the QSFPs spawn {18..25}, {26..33}, {34..41} and {42..49}
+                      # starting at chennel 18 and 8 channels per mux.
+                      channel_first=18
                       for ((i=9;i>=6;i--));
                       do
                           # 0x71 mux on the IOM 1
                           mux_index=$(expr $i - 5)
                           echo "Attaching PCA9548 $mux_index"
-                          i2c_config "echo pca9548 0x71 > /sys/bus/i2c/devices/i2c-$i/$1"
+                          i2c_mux_create pca9548 0x71 $i $channel_first
+                          channel_first=$(expr $channel_first + 8)
                       done
                       ;;
         "delete_device")
@@ -92,7 +96,7 @@ switch_board_qsfp_mux() {
                           # 0x71 mux on the IOM 1
                           mux_index=$(expr $i - 5)
                           echo "Detaching PCA9548 $mux_index"
-                          i2c_config "echo 0x71 > /sys/bus/i2c/devices/i2c-$devnum/i2c-$i/$1"
+                          i2c_mux_delete 0x71 $i
                       done
                       ;;
         *)            echo "z9100_platform: switch_board_qsfp_mux: invalid command !"
@@ -156,6 +160,17 @@ xcvr_presence_interrupts() {
     esac
 }
 
+# Reset the mux tree
+reset_muxes() {
+    # Reset the IOM muxes and the switch card mux
+    io_rd_wr.py --set --val 0xe0 --offset 0x110
+    io_rd_wr.py --set --val 0xff --offset 0x110
+
+    # Reset the CPU Card PCA9547
+    io_rd_wr.py --set --val 0xfd --offset 0x20b
+    io_rd_wr.py --set --val 0xff --offset 0x20b
+}
+
 init_devnum
 
 if [[ "$1" == "init" ]]; then