From 20ecb7a2c12e146f0c2f098e324ca18f664a11fc Mon Sep 17 00:00:00 2001 From: Prince George Date: Thu, 2 Oct 2025 17:46:18 +0000 Subject: [PATCH] FEC histogram with ability to clear stat --- doc/Command-Reference.md | 28 +++++++++++++++++++++++ scripts/portstat | 24 +++++++++++++++----- utilities_common/portstat.py | 44 ++++++++++++++++++++++++++++++++++-- 3 files changed, 88 insertions(+), 8 deletions(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 57ffc77a50..66cc244de6 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -5656,6 +5656,34 @@ The "fec-stats" subcommand is used to disply the interface fec related statistic Ethernet16 U 0 0 0 1.77e-20 0.00e+00 1.37e-13 ``` +For debugging link related issues where you need to clear the FEC histogram and monitor the link again, use the following command + +- Example (for all ports): + ``` + root@sonic:~# portstat -fh + Last cached time was 2025-10-02T16:43:57.934081 + IFACE BIN0 BIN1 BIN2 BIN3 BIN4 BIN5 BIN6 BIN7 BIN8 BIN9 BIN10 BIN11 BIN12 BIN13 BIN14 BIN15 + ----------- ------------- ---------- --------- ------ ------ ------ ------ ------ ------ ------ ------- ------- ------- ------- ------- ------- + Ethernet0 4,374,661,575 340 1 0 0 0 0 0 0 0 0 0 0 0 0 0 + Ethernet8 4,374,590,263 8,069 9 0 0 0 0 0 0 0 0 0 0 0 0 0 + Ethernet16 4,374,660,911 3,187 4 0 0 0 0 0 0 0 0 0 0 0 0 0 + Ethernet24 4,374,594,305 57,484 502 0 0 0 0 0 0 0 0 0 0 0 0 0 + Ethernet32 4,374,649,615 116 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + Ethernet40 4,374,650,913 1,212 1 0 0 0 0 0 0 0 0 0 0 0 0 0 + ``` + + - Example (for a particular port): + ``` + root@sonic:~# portstat -fh -i Ethernet504 + Last cached time was 2025-10-02T16:43:57.934081 + IFACE BIN0 BIN1 BIN2 BIN3 BIN4 BIN5 BIN6 BIN7 BIN8 BIN9 BIN10 BIN11 BIN12 BIN13 BIN14 BIN15 + ----------- ----------- ------ ------ ------ ------ ------ ------ ------ ------ ------ ------- ------- ------- ------- ------- ------- + Ethernet504 624,891,017 13,331 172 0 0 0 0 0 0 0 0 0 0 0 0 0 + root@str-7060x6-c09-u25:~# + ``` + + To clear the FEC histogram use `portstat -c`. NOTE: This will clear all counters. + The "trim" subcommand is used to display the interface packet trimming related statistic. - Example: diff --git a/scripts/portstat b/scripts/portstat index 7583fbe0c8..56aae631fe 100755 --- a/scripts/portstat +++ b/scripts/portstat @@ -70,6 +70,7 @@ Examples: parser.add_argument('-D', '--delete-all', action='store_true', help='Delete all saved stats') parser.add_argument('-e', '--errors', action='store_true', help='Display interface errors') parser.add_argument('-f', '--fec-stats', action='store_true', help='Display FEC related statistics') + parser.add_argument('-fh', '--fec_hist', action='store_true', help='Display FEC histogram') parser.add_argument('-j', '--json', action='store_true', help='Display in JSON format') parser.add_argument('-r', '--raw', action='store_true', help='Raw stats (unmodified output of netstat)') parser.add_argument('-R', '--rate', action='store_true', help='Display interface rates') @@ -89,6 +90,7 @@ Examples: delete_all_stats = args.delete_all errors_only = args.errors fec_stats_only = args.fec_stats + fec_hist_only = args.fec_hist rates_only = args.rate use_json = args.json raw_stats = args.raw @@ -127,8 +129,10 @@ Examples: # Now decide what information to display if raw_stats: - portstat.cnstat_diff_print(cnstat_dict, {}, ratestat_dict, intf_list, use_json, print_all, errors_only, - fec_stats_only, rates_only, trim_stats_only) + portstat.cnstat_diff_print(cnstat_dict, {}, ratestat_dict, intf_list, + use_json, print_all, errors_only, + fec_stats_only, rates_only, + trim_stats_only, fec_hist_only) sys.exit(0) if save_fresh_stats: @@ -147,7 +151,10 @@ Examples: cnstat_cached_dict = json.load(open(cnstat_fqn_file, 'r')) if not detail: print("Last cached time was " + str(cnstat_cached_dict.get('time'))) - portstat.cnstat_diff_print(cnstat_dict, cnstat_cached_dict, ratestat_dict, intf_list, use_json, print_all, errors_only, fec_stats_only, rates_only, trim_stats_only, detail, nonzero) + portstat.cnstat_diff_print(cnstat_dict, cnstat_cached_dict, ratestat_dict, + intf_list, use_json, print_all, errors_only, + fec_stats_only, rates_only, trim_stats_only, + fec_hist_only, detail, nonzero) except IOError as e: print(e.errno, e) else: @@ -155,14 +162,19 @@ Examples: print("\nFile '%s' does not exist" % cnstat_fqn_file) print("Did you run 'portstat -c -t %s' to record the counters via tag %s?\n" % (tag_name, tag_name)) else: - portstat.cnstat_diff_print(cnstat_dict, {}, ratestat_dict, intf_list, use_json, print_all, errors_only, - fec_stats_only, rates_only, trim_stats_only, detail, nonzero) + portstat.cnstat_diff_print(cnstat_dict, {}, ratestat_dict, intf_list, + use_json, print_all, errors_only, + fec_stats_only, rates_only, trim_stats_only, + fec_hist_only, detail, nonzero) else: #wait for the specified time and then gather the new stats and output the difference. time.sleep(wait_time_in_seconds) print("The rates are calculated within %s seconds period" % wait_time_in_seconds) cnstat_new_dict, ratestat_new_dict = portstat.get_cnstat_dict() - portstat.cnstat_diff_print(cnstat_new_dict, cnstat_dict, ratestat_new_dict, intf_list, use_json, print_all, errors_only, fec_stats_only, rates_only, trim_stats_only, detail, nonzero) + portstat.cnstat_diff_print(cnstat_new_dict, cnstat_dict, ratestat_new_dict, + intf_list, use_json, print_all, errors_only, + fec_stats_only, rates_only, trim_stats_only, + fec_hist_only, detail, nonzero) if __name__ == "__main__": main() diff --git a/utilities_common/portstat.py b/utilities_common/portstat.py index 54e95f2cf7..8ec5387c73 100644 --- a/utilities_common/portstat.py +++ b/utilities_common/portstat.py @@ -29,7 +29,9 @@ rx_jbr, rx_frag, rx_usize, rx_ovrrun,\ fec_corr, fec_uncorr, fec_symbol_err,\ wred_grn_drp_pkt, wred_ylw_drp_pkt, wred_red_drp_pkt, wred_tot_drp_pkt,\ - trim, trim_sent, trim_drop") + trim, trim_sent, trim_drop, fec_bin0, fec_bin1, fec_bin2, fec_bin3,\ + fec_bin4, fec_bin5, fec_bin6, fec_bin7, fec_bin8, fec_bin9, fec_bin10,\ + fec_bin11, fec_bin12, fec_bin13, fec_bin14, fec_bin15") header_all = ['IFACE', 'STATE', 'RX_OK', 'RX_BPS', 'RX_PPS', 'RX_UTIL', 'RX_ERR', 'RX_DRP', 'RX_OVR', 'TX_OK', 'TX_BPS', 'TX_PPS', 'TX_UTIL', 'TX_ERR', 'TX_DRP', 'TX_OVR', 'TRIM', 'TRIM_TX', 'TRIM_DRP'] header_std = ['IFACE', 'STATE', 'RX_OK', 'RX_BPS', 'RX_UTIL', 'RX_ERR', 'RX_DRP', 'RX_OVR', @@ -37,6 +39,8 @@ header_errors_only = ['IFACE', 'STATE', 'RX_ERR', 'RX_DRP', 'RX_OVR', 'TX_ERR', 'TX_DRP', 'TX_OVR'] header_fec_only = ['IFACE', 'STATE', 'FEC_CORR', 'FEC_UNCORR', 'FEC_SYMBOL_ERR', 'FEC_PRE_BER', 'FEC_POST_BER', 'FEC_PRE_BER_MAX'] +header_fec_hist_only = ['IFACE', 'BIN0', 'BIN1', 'BIN2', 'BIN3', 'BIN4', 'BIN5', 'BIN6', 'BIN7', + 'BIN8', 'BIN9', 'BIN10', 'BIN11', 'BIN12', 'BIN13', 'BIN14', 'BIN15'] header_rates_only = ['IFACE', 'STATE', 'RX_OK', 'RX_BPS', 'RX_PPS', 'RX_UTIL', 'TX_OK', 'TX_BPS', 'TX_PPS', 'TX_UTIL'] header_trim_only = ['IFACE', 'STATE', 'TRIM_PKTS', 'TRIM_TX_PKTS', 'TRIM_DRP_PKTS'] @@ -112,6 +116,22 @@ 49: ['SAI_PORT_STAT_TRIM_PACKETS'], 50: ['SAI_PORT_STAT_TX_TRIM_PACKETS'], 51: ['SAI_PORT_STAT_DROPPED_TRIM_PACKETS'], + 52: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S0'], + 53: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S1'], + 54: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S2'], + 55: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S3'], + 56: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S4'], + 57: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S5'], + 58: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S6'], + 59: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S7'], + 60: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S8'], + 61: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S9'], + 62: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S10'], + 63: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S11'], + 64: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S12'], + 65: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S13'], + 66: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S14'], + 67: ['SAI_PORT_STAT_IF_IN_FEC_CODEWORD_ERRORS_S15'], } STATUS_NA = 'N/A' @@ -550,7 +570,7 @@ def cnstat_intf_diff_print(self, cnstat_new_dict, cnstat_old_dict, intf_list): def cnstat_diff_print(self, cnstat_new_dict, cnstat_old_dict, ratestat_dict, intf_list, use_json, print_all, errors_only, fec_stats_only, - rates_only, trim_stats_only, detail=False, nonzero=False): + rates_only, trim_stats_only, fec_hist_only, detail=False, nonzero=False): """ Print the difference between two cnstat results. """ @@ -638,6 +658,26 @@ def cnstat_diff_print(self, cnstat_new_dict, cnstat_old_dict, format_fec_ber(rates.fec_pre_ber), format_fec_ber(rates.fec_post_ber), format_fec_ber(rates.fec_pre_ber_max))) + elif fec_hist_only: + header = header_fec_hist_only + + table.append((key, ns_diff(cntr['fec_bin0'], old_cntr['fec_bin0']), + ns_diff(cntr['fec_bin1'], old_cntr['fec_bin1']), + ns_diff(cntr['fec_bin2'], old_cntr['fec_bin2']), + ns_diff(cntr['fec_bin3'], old_cntr['fec_bin3']), + ns_diff(cntr['fec_bin4'], old_cntr['fec_bin4']), + ns_diff(cntr['fec_bin5'], old_cntr['fec_bin5']), + ns_diff(cntr['fec_bin6'], old_cntr['fec_bin6']), + ns_diff(cntr['fec_bin7'], old_cntr['fec_bin7']), + ns_diff(cntr['fec_bin8'], old_cntr['fec_bin8']), + ns_diff(cntr['fec_bin9'], old_cntr['fec_bin9']), + ns_diff(cntr['fec_bin10'], old_cntr['fec_bin10']), + ns_diff(cntr['fec_bin11'], old_cntr['fec_bin11']), + ns_diff(cntr['fec_bin12'], old_cntr['fec_bin12']), + ns_diff(cntr['fec_bin13'], old_cntr['fec_bin13']), + ns_diff(cntr['fec_bin14'], old_cntr['fec_bin14']), + ns_diff(cntr['fec_bin15'], old_cntr['fec_bin15']))) + elif rates_only: header = header_rates_only