8000 Fix for Recognizing --threads Option in Benchmarking by sujik18 · Pull Request #2161 · mlcommons/inference · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Fix for Recognizing --threads Option in Benchmarking #2161

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: dev
Choose a base branch
from
Open

Fix for Recognizing --threads Option in Benchmarking #2161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8e4fb02
Update backend_onnxruntime.py
sujik18 Mar 28, 2025
28c3140
Update backend_onnxruntime.py
sujik18 Mar 28, 2025
a8b8305
Update main.py
sujik18 Mar 28, 2025
6dc16bf
load function thread parameter update
sujik18 Mar 30, 2025
edf0ac3
[Automated Commit] Format Codebase
github-actions[bot] Mar 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion vision/classification_and_detection/python/backend.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ def version(self):
def name(self):
raise NotImplementedError("Backend:name")

def load(self, model_path, inputs=None, outputs=None):
def load(self, model_path, inputs=None, outputs=None, threads=None):
raise NotImplementedError("Backend:load")

def predict(self, feed):
Expand Down
2 changes: 1 addition & 1 deletion vision/classification_and_detection/python/backend_ncnn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def image_format(self):
"""image_format. For ncnn it is NCHW."""
return "NCHW"

def load(self, model_path, inputs=None, outputs=None):
def load(self, model_path, inputs=None, outputs=None, threads=None):
param_file, bin_file = f"{model_path}.param", f"{model_path}.bin"
if param_file.endswith("resnet50_v1.param"):
# download model files if doesn't
Expand Down
7 changes: 6 additions & 1 deletion vision/classification_and_detection/python/backend_onnxruntime.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,14 @@ def image_format(self):
"""image_format. For onnx it is always NCHW."""
return "NCHW"

def load(self, model_path, inputs=None, outputs=None):
def load(self, model_path, inputs=None, outputs=None, threads=None):
"""Load model and find input/outputs from the model file."""
print("************************************************************")
print(">>> Value of num_threads: ", threads)
print("************************************************************")
opt = rt.SessionOptions()
opt.intra_op_num_threads = int(threads)
opt.inter_op_num_threads = int(threads)

# By default all optimizations are enabled
# https://onnxruntime.ai/docs/performance/graph-optimizations.html
Expand Down
4 changes: 3 additions & 1 deletion vision/classification_and_detection/python/backend_pytorch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,10 @@ def name(self):
def image_format(self):
return "NCHW"

def load(self, model_path, inputs=None, outputs=None):
def load(self, model_path, inputs=None, outputs=None, threads=None):
self.model = onnx.load(model_path)
torch.set_num_threads(threads)
torch.set_num_interop_threads(threads)

# find inputs from the model if not passed in by config
if inputs:
Expand Down
4 changes: 3 additions & 1 deletion vision/classification_and_detection/python/backend_pytorch_native.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,11 @@ def name(self):
def image_format(self):
return "NCHW"

def load(self, model_path, inputs=None, outputs=None):
def load(self, model_path, inputs=None, outputs=None, threads=None):
self.model = torch.load(model_path, weights_only=False)
self.model.eval()
torch.set_num_threads(threads)
torch.set_num_interop_threads(threads)
# find inputs from the model if not passed in by config
if inputs:
self.inputs = inputs
Expand Down
2 changes: 1 addition & 1 deletion vision/classification_and_detection/python/backend_tf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ def image_format(self):
# NHWC)
return "NHWC"

def load(self, model_path, inputs=None, outputs=None):
def load(self, model_path, inputs=None, outputs=None, threads=None):
# there is no input/output meta data i the graph so it need to come
# from config.
if not inputs:
Expand Down
2 changes: 1 addition & 1 deletion vision/classification_and_detection/python/backend_tvm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ def _worker_handler(feed):
global global_executor
return global_executor.predict(feed)

def load(self, model_path, inputs=None, outputs=None):
def load(self, model_path, inputs=None, outputs=None, threads=None):
"""Load model and find input/outputs from the model file."""
self.load_impl(model_path, inputs, outputs, self.max_batchsize)

Expand Down
16 changes: 15 additions & 1 deletion vision/classification_and_detection/python/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,18 @@ def get_args():

if args.scenario not in SCENARIO_MAP:
parser.error("valid scanarios:" + str(list(SCENARIO_MAP.keys())))

if args.threads:
num_threads = args.threads
print("*****>>> Value of num_threads: {num_threads} **********")
os.environ["OMP_NUM_THREADS"] = str(num_threads)
os.environ["OPENBLAS_NUM_THREADS"] = str(num_threads)
os.environ["MKL_NUM_THREADS"] = str(num_threads)
os.environ["TF_NUM_INTRAOP_THREADS"] = str(num_threads)
os.environ["TF_NUM_INTEROP_THREADS"] = str(num_threads)
os.environ["TVM_NUM_THREADS"] = str(num_threads)
os.environ["TFLITE_NUM_THREADS"] = str(num_threads)
os.environ["NCNN_NUM_THREADS"] = str(num_threads)
return args


Expand Down Expand Up @@ -646,7 +658,9 @@ def main():
model = backend.load(
args.model,
inputs=args.inputs,
outputs=args.outputs)
outputs=args.outputs,
threads=args.threads,
)
final_results = {
"runtime": model.name(),
"version": model.version(),
Expand Down
0