Switch to Python 3 [Backport of #9 to 6.0.3.0] (#43)

Don Brady · gllghr · web-flow · commit 607e52ffdd07 · 2020-05-19T16:24:30.000-06:00
Co-authored-by: John Gallagher &lt;john.gallagher@delphix.com&gt;
diff --git a/.github/scripts/flake8_stbtrace.sh b/.github/scripts/flake8_stbtrace.sh
@@ -0,0 +1,33 @@
+#/bin/bash
+#
+# Copyright (c) 2020 by Delphix. All rights reserved.
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+
+#
+# This provides flake8 linting for the various stbtrace templates. It uses
+# stbtrace with the '--bcc' flag to produce and print Python code, and then
+# runs flake8 against that.
+#
+
+set -o pipefail
+
+function die() {
+        echo "$(basename "$0"): $*" >&2
+        exit 1
+}
+
+prgms=$(find ./bpf/stbtrace -name '*.st' | xargs basename --suffix='.st') ||
+	die "Failed to generate list of stbtrace programs"
+
+for prgm in $prgms; do
+	echo "Checking stbtrace program $prgm..."
+	./cmd/stbtrace.py $prgm --bcc >/tmp/$prgm.py ||
+		die "Failed to generate python source"
+	flake8 /tmp/$prgm.py --show-source || die "flake8 errors found in" \
+		"'bpf/stbtrace/$prgm.st'. Line numbers in error messages" \
+		"correspond to the output of './cmd/stbtrace.py $prgm --bcc'"
+done
+
+exit 0
diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml
@@ -5,15 +5,15 @@ on: [pull_request, push]
 jobs:
   build:
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python
       uses: actions/setup-python@v1
       with:
-        python-version: 2.7
+        python-version: 3.6
     - name: flake8
       run: |
         python -m pip install --upgrade pip
         pip install flake8
         flake8 . --show-source --statistics
+        ./.github/scripts/flake8_stbtrace.sh
diff --git a/bpf/standalone/zil.py b/bpf/standalone/zil.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 #
 # Copyright 2019 Delphix. All rights reserved.
 #
@@ -305,7 +305,7 @@
         exit(0)
 
 # Collect data until keyborad interrupt with output for each second
-while (1):
+while True:
     try:
         sleep(60)
     except KeyboardInterrupt:
diff --git a/bpf/stbtrace/io.st b/bpf/stbtrace/io.st
@@ -6,11 +6,9 @@
 
 from bcc import BPF
 from time import sleep
-from time import time
-import datetime
-import ctypes as ct
 import os
 import sys
+
 #
 # Find BCCHelper. If we are being run from the repo, we should be able to find
 # it in the repo's lib/ directory. If we can't find that, look for BCCHelper
@@ -20,7 +18,7 @@ base_dir = os.path.dirname(__file__) + "/../"
 if not os.path.exists(base_dir + "lib/bcchelper.py"):
     base_dir = "/usr/share/performance-diagnostics/"
 sys.path.append(base_dir + 'lib/')
-from bcchelper import BCCHelper
+from bcchelper import BCCHelper   # noqa: E402
 
 
 # BPF disk io program
@@ -125,7 +123,7 @@ int disk_io_done(struct pt_regs *ctx, struct request *reqp)
     io_base_data.delete((u64 *) &reqp);
     return 0;
 }
-"""
+"""  # noqa: W293
 b = BPF(text=bpf_text)
 
 if BPF.get_kprobe_functions(b'blk_start_request'):
diff --git a/bpf/stbtrace/iscsi.st b/bpf/stbtrace/iscsi.st
@@ -6,11 +6,9 @@
 
 from bcc import BPF
 from time import sleep
-from time import time
-import datetime
-import ctypes as ct
 import os
 import sys
+
 #
 # Find BCCHelper. If we are being run from the repo, we should be able to find
 # it in the repo's lib/ directory. If we can't find that, look for BCCHelper
@@ -20,7 +18,7 @@ base_dir = os.path.dirname(__file__) + "/../"
 if not os.path.exists(base_dir + "lib/bcchelper.py"):
     base_dir = "/usr/share/performance-diagnostics/"
 sys.path.append(base_dir + 'lib/')
-from bcchelper import BCCHelper
+from bcchelper import BCCHelper   # noqa: E402
 
 # BPF txg program
 bpf_text = """
@@ -63,7 +61,8 @@ BPF_HASH($hist.name$, iscsi_hist_key_t, u64);
 }$
 
 // Probe functions to initialize thread local data
-int iscsi_target_start(struct pt_regs *ctx, struct iscsi_conn *conn, struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr)
+int iscsi_target_start(struct pt_regs *ctx, struct iscsi_conn *conn,
+                       struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr)
 {
     iscsi_data_t data = {};
     data.ts = bpf_ktime_get_ns();
@@ -122,7 +121,7 @@ int iscsi_target_end(struct pt_regs *ctx, struct iscsi_cmd *cmd)
     return 0;
 }
 
-"""
+"""  # noqa: W293
 b = BPF(text=bpf_text)
 
 b.attach_kprobe(event="iscsit_process_scsi_cmd", fn_name="iscsi_target_start")
diff --git a/bpf/stbtrace/nfs.st b/bpf/stbtrace/nfs.st
@@ -6,11 +6,9 @@
 
 from bcc import BPF
 from time import sleep
-from time import time
-import datetime
-import ctypes as ct
 import os
 import sys
+
 #
 # Find BCCHelper. If we are being run from the repo, we should be able to find
 # it in the repo's lib/ directory. If we can't find that, look for BCCHelper
@@ -20,7 +18,7 @@ base_dir = os.path.dirname(__file__) + "/../"
 if not os.path.exists(base_dir + "lib/bcchelper.py"):
     base_dir = "/usr/share/performance-diagnostics/"
 sys.path.append(base_dir + 'lib/')
-from bcchelper import BCCHelper
+from bcchelper import BCCHelper   # noqa: E402
 
 # BPF txg program
 bpf_text = """
@@ -167,8 +165,8 @@ int nfsd4_read_start(struct pt_regs *ctx, struct svc_rqst *rqstp, void *cstate,
     return 0;
 }
 
-int nfsd4_write_start(struct pt_regs *ctx, struct svc_rqst *rqstp, void *cstate,
-    nfsd4_write *nfs_write)
+int nfsd4_write_start(struct pt_regs *ctx, struct svc_rqst *rqstp,
+    void *cstate, nfsd4_write *nfs_write)
 {
     u32 pid = bpf_get_current_pid_tgid();
     nfs_data_t data = {};
@@ -302,7 +300,7 @@ int nfsd4_write_done(struct pt_regs *ctx)
 
     return 0;
 }
-"""
+"""  # noqa: W293
 b = BPF(text=bpf_text)
 
 b.attach_kprobe(event="nfsd_read", fn_name="nfsd3_read_start")
diff --git a/bpf/stbtrace/vfs.st b/bpf/stbtrace/vfs.st
@@ -6,11 +6,9 @@
 
 from bcc import BPF
 from time import sleep
-from time import time
-import datetime
-import ctypes as ct
 import os
 import sys
+
 #
 # Find BCCHelper. If we are being run from the repo, we should be able to find
 # it in the repo's lib/ directory. If we can't find that, look for BCCHelper
@@ -20,7 +18,7 @@ base_dir = os.path.dirname(__file__) + "/../"
 if not os.path.exists(base_dir + "lib/bcchelper.py"):
     base_dir = "/usr/share/performance-diagnostics/"
 sys.path.append(base_dir + 'lib/')
-from bcchelper import BCCHelper
+from bcchelper import BCCHelper   # noqa: E402
 
 # BPF txg program
 bpf_text = """
@@ -68,7 +66,7 @@ BPF_HASH($hist.name$, vfs_hist_key_t, u64);
 
 // Probe functions to initialize thread local data
 int vfs_start(struct pt_regs *ctx, void *file, void *user_buf,
-			       u64 count, void *ppos)
+    u64 count, void *ppos)
 {
     u32 pid = bpf_get_current_pid_tgid();
     vfs_data_t data = {};
@@ -187,7 +185,7 @@ int vfs_write_done(struct pt_regs *ctx)
     u64 ts = bpf_ktime_get_ns();
     return vfs_aggregate_data(ts, WRITE_STR);
 }
-"""
+"""  # noqa: W293
 b = BPF(text=bpf_text)
 
 b.attach_kprobe(event="vfs_read", fn_name="vfs_start")
diff --git a/bpf/stbtrace/zio.st b/bpf/stbtrace/zio.st
@@ -6,11 +6,9 @@
 
 from bcc import BPF
 from time import sleep
-from time import time
-import datetime
-import ctypes as ct
 import os
 import sys
+
 #
 # Find BCCHelper. If we are being run from the repo, we should be able to find
 # it in the repo's lib/ directory. If we can't find that, look for BCCHelper
@@ -20,7 +18,7 @@ base_dir = os.path.dirname(__file__) + "/../"
 if not os.path.exists(base_dir + "lib/bcchelper.py"):
     base_dir = "/usr/share/performance-diagnostics/"
 sys.path.append(base_dir + 'lib/')
-from bcchelper import BCCHelper
+from bcchelper import BCCHelper   # noqa: E402
 
 # BPF txg program
 bpf_text = """
@@ -125,7 +123,7 @@ int vdev_queue_done(struct pt_regs *ctx, zio_t *zio)
 }
 
 
-"""
+"""  # noqa: W293
 
 KVER = os.popen('uname -r').read().rstrip()
 b = BPF(text=bpf_text, cflags=["-include",
@@ -135,7 +133,8 @@ b = BPF(text=bpf_text, cflags=["-include",
                                "-I/usr/src/zfs-" + KVER + "/include/linux",
                                "-DCC_USING_FENTRY"])
 
-b.attach_kretprobe(event="vdev_queue_io_to_issue", fn_name="vdev_queue_issue_return")
+b.attach_kretprobe(event="vdev_queue_io_to_issue",
+                   fn_name="vdev_queue_issue_return")
 b.attach_kprobe(event="vdev_queue_io_done", fn_name="vdev_queue_done")
 
 helper = BCCHelper(b, BCCHelper.ANALYTICS_PRINT_MODE)
diff --git a/bpf/stbtrace/zpl.st b/bpf/stbtrace/zpl.st
@@ -6,11 +6,9 @@
 
 from bcc import BPF
 from time import sleep
-from time import time
-import datetime
-import ctypes as ct
 import os
 import sys
+
 #
 # Find BCCHelper. If we are being run from the repo, we should be able to find
 # it in the repo's lib/ directory. If we can't find that, look for BCCHelper
@@ -20,7 +18,8 @@ base_dir = os.path.dirname(__file__) + "/../"
 if not os.path.exists(base_dir + "lib/bcchelper.py"):
     base_dir = "/usr/share/performance-diagnostics/"
 sys.path.append(base_dir + 'lib/')
-from bcchelper import BCCHelper
+from bcchelper import BCCHelper   # noqa: E402
+
 
 # BPF txg program
 bpf_text = """
@@ -170,7 +169,8 @@ int zfs_write_done(struct pt_regs *ctx)
     return zpl_aggregate_data(ts, WRITE_STR);
 }
 
-"""
+"""  # noqa: W293
+
 KVER = os.popen('uname -r').read().rstrip()
 b = BPF(text=bpf_text,
         cflags=["-I/usr/src/zfs-" + KVER + "/include/spl/",
diff --git a/cmd/estat.py b/cmd/estat.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 #
 # Copyright 2019 Delphix. All rights reserved.
 #
@@ -19,8 +19,6 @@
 #   the performance data collector.
 #
 
-from __future__ import print_function
-
 from bcc import BPF
 import getopt
 from glob import glob
@@ -423,7 +421,7 @@ class Args:
                 line + "'")
         probe_type = probe_spec[0]
         if probe_type == "kprobe":
-            if BPF.get_kprobe_functions(probe_spec[1]):
+            if BPF.get_kprobe_functions(probe_spec[1].encode('utf-8')):
                 b.attach_kprobe(event=probe_spec[1], fn_name=probe_spec[2])
                 probes.add("p_" + probe_spec[1] + "_bcc_" + str(os.getpid()))
             else:
@@ -481,20 +479,20 @@ class Args:
 # output
 if monitor:
     # TODO can we do this without shelling out to 'date'?
-    ds__start = long(os.popen("date +%s%N").readlines()[0])
+    ds__start = int(os.popen("date +%s%N").readlines()[0])
     while (1):
         try:
             sleep(duration)
         except KeyboardInterrupt:
             break
         try:
-            ds__end = long(os.popen("date +%s%N").readlines()[0])
+            ds__end = int(os.popen("date +%s%N").readlines()[0])
             ds__delta = ds__end - ds__start
             if not accum:
                 ds__start = ds__end
             if args.summary and args.normalize:
-                helper1.normalize("ops", ds__delta / 1000000000)
-                helper3.normalize("opst", ds__delta / 1000000000)
+                helper1.normalize("ops", ds__delta // 1000000000)
+                helper3.normalize("opst", ds__delta // 1000000000)
             clear_data = not accum
             if args.latsize_hist:
                 helper2.printall(clear_data)
@@ -503,26 +501,26 @@ class Args:
             if args.summary and args.total:
                 helper3.printall(clear_data)
             print("%-16s\n" % strftime("%D - %H:%M:%S %Z"))
-        except e:
+        except Exception as e:
             die(e)
 else:
-    ds__start = long(os.popen("date +%s%N").readlines()[0])
+    ds__start = int(os.popen("date +%s%N").readlines()[0])
     try:
         sleep(duration)
     except KeyboardInterrupt:
         pass
     try:
-        ds__delta = long(os.popen("date +%s%N").readlines()[0]) - ds__start
+        ds__delta = int(os.popen("date +%s%N").readlines()[0]) - ds__start
         if args.summary and args.normalize:
-            helper1.normalize("ops", ds__delta / 1000000000)
-            helper3.normalize("opst", ds__delta / 1000000000)
+            helper1.normalize("ops", ds__delta // 1000000000)
+            helper3.normalize("opst", ds__delta // 1000000000)
         if args.latsize_hist:
             helper2.printall()
         if args.lat_hist or args.size_hist or args.summary:
             helper1.printall()
         if args.summary and args.total:
             helper3.printall()
-    except e:
+    except Exception as e:
         die(e)
 
 #
diff --git a/cmd/stbtrace.py b/cmd/stbtrace.py
diff --git a/debian/control b/debian/control
diff --git a/lib/bcchelper.py b/lib/bcchelper.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-#!/usr/bin/env python2`
	`1`	`+#!/usr/bin/env python3`
`2`	`2`	`#`
`3`	`3`	`# Copyright 2019 Delphix. All rights reserved.`
`4`	`4`	`#`
`@@ -305,7 +305,7 @@`
`305`	`305`	`exit(0)`
`306`	`306`
`307`	`307`	`# Collect data until keyborad interrupt with output for each second`
`308`		`-while (1):`
	`308`	`+while True:`
`309`	`309`	`try:`
`310`	`310`	`sleep(60)`
`311`	`311`	`except KeyboardInterrupt:`