Load in C++

See https://pytorch.org/tutorials/advanced/cpp_export.html.

Load the saved foo.pt in C++ from Export and ignore methods.

./code/load-in-cpp/Makefile

USE_CXX11_ABI := $(shell python3 -c 'import torch; print(int(torch.compiled_with_cxx11_abi()))')
TORCH_INSTALL_DIR := $(shell python3 -c 'import os; import torch; print(os.path.dirname(torch.__file__))')

$(info USE_CXX11_ABI $(USE_CXX11_ABI))
$(info TORCH_INSTALL_DIR $(TORCH_INSTALL_DIR))

CXXFLAGS := -I$(TORCH_INSTALL_DIR)/include
CXXFLAGS += -I$(TORCH_INSTALL_DIR)/include/torch/csrc/api/include
CXXFLAGS += -I$(TORCH_INSTALL_DIR)/include/TH
CXXFLAGS += -I$(TORCH_INSTALL_DIR)/include/THC
CXXFLAGS += -std=c++14
CXXFLAGS += -D_GLIBCXX_USE_CXX11_ABI=$(USE_CXX11_ABI)

CXXFLAGS += -Wno-unknown-pragmas # disable omp warnings

LDFLAGS := -L$(TORCH_INSTALL_DIR)/lib
LDFLAGS += -lc10 -ltorch -ltorch_cpu
# LDFLAGS += -lc10 -ltorch
LDFLAGS += -Wl,-rpath,$(TORCH_INSTALL_DIR)/lib

HAS_CUDA := $(shell python3 -c 'import torch; print("yes" if torch.cuda.is_available() else "no")')
HAS_CUDA := yes
$(info has cuda $(HAS_CUDA))

ifeq ($(HAS_CUDA),yes)
CUDA_HOME := $(shell which nvcc | xargs dirname | xargs dirname)
CXXFLAGS += -I$(CUDA_HOME)/include
LDFLAGS += -L$(CUDA_HOME)/lib64
LDFLAGS += -lcudart -lc10_cuda -ltorch_cuda
LDFLAGS += -Wl,-rpath,$(CUDA_HOME)/lib64
endif

.PHONY: clean

main: main.o
	$(CXX) -o $@ $< $(LDFLAGS)

main.o: main.cc
	$(CXX) $(CXXFLAGS) -c -o $@ $<

clean:
	$(RM) main.o main

Note

torch::jit::script::Module is deprecated, use torch::jit::Module instead.

./code/load-in-cpp/main.cc

#include "torch/script.h"

int main() {
  // see torch/csrc/jit/module.h
  torch::jit::Module m = torch::jit::load("../foo.pt");
  std::cout << "is training: " << m.is_training() << "\n";
  m.eval();
  std::cout << "after m.eval(): is training: " << m.is_training() << "\n";
  torch::Tensor x = torch::tensor({1, 2, 3}, torch::kFloat);
  torch::Tensor y = m.run_method("baz", x).toTensor();
  std::cout << y << "\n";

  return 0;
}

The output of make is:

USE_CXX11_ABI 0
TORCH_INSTALL_DIR /ceph-fj/fangjun/software/py38/lib/python3.8/site-packages/torch
has cuda yes
g++ -I/ceph-fj/fangjun/software/py38/lib/python3.8/site-packages/torch/include \
    -I/ceph-fj/fangjun/software/py38/lib/python3.8/site-packages/torch/include/torch/csrc/api/include \
    -I/ceph-fj/fangjun/software/py38/lib/python3.8/site-packages/torch/include/TH \
    -I/ceph-fj/fangjun/software/py38/lib/python3.8/site-packages/torch/include/THC \
    -std=c++14 \
    -D_GLIBCXX_USE_CXX11_ABI=0 \
    -Wno-unknown-pragmas \
    -I/ceph-sh1/fangjun/software/cuda-10.2.89/include \
    -c -o main.o main.cc
g++ -o main main.o \
    -L/ceph-fj/fangjun/software/py38/lib/python3.8/site-packages/torch/lib \
    -lc10 -ltorch -ltorch_cpu \
    -Wl,-rpath,/ceph-fj/fangjun/software/py38/lib/python3.8/site-packages/torch/lib \
    -L/ceph-sh1/fangjun/software/cuda-10.2.89/lib64 \
    -lcudart -lc10_cuda -ltorch_cuda \
    -Wl,-rpath,/ceph-sh1/fangjun/software/cuda-10.2.89/lib64

The output of ./main is:

is training: 1
after m.eval(): is training: 0
 5
 6
 7
[ CPUFloatType{3} ]