Jetcam library doesn't work every time, so using OpenCV to open camera

[SarthakGarg19]

Using OpenCV to open camera instead of jetcam because many a times jetcam faces issues with USB camera and fails to initialise the camera.
Also adding a python script named live_demo.py, which can be directly run for inference by typing "python3 live_demo.py"

[kdy1999]

awesome!!!

[lbq779660843]

Hi@SarthakGarg13 ,
So many thanks foryou share this script and I change some parts because my nano will throw an error while I run yours directly, this is the new one I use now.

###################################################################
`import json
import trt_pose.coco

import torch
import torch2trt
from torch2trt import TRTModule

import cv2
import torchvision.transforms as transforms
import PIL.Image

from trt_pose.draw_objects import DrawObjects
from trt_pose.parse_objects import ParseObjects

WIDTH = 1280
HEIGHT = 720
OPTIMIZED_MODEL = 'resnet18_baseline_att_224x224_A_epoch_249_trt.pth'

with open('human_pose.json', 'r') as f:
human_pose = json.load(f)

topology = trt_pose.coco.coco_category_to_topology(human_pose)

num_parts = len(human_pose['keypoints'])
num_links = len(human_pose['skeleton'])

print("Reading TensorRT model.......")
model_trt = TRTModule()
model_trt.load_state_dict(torch.load(OPTIMIZED_MODEL))

print("TensorRT model loaded")

mean = torch.Tensor([0.485, 0.456, 0.406]).cuda()
std = torch.Tensor([0.229, 0.224, 0.225]).cuda()
device = torch.device('cuda')

def preprocess(image):
global device
device = torch.device('cuda')
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
image = PIL.Image.fromarray(image)
image = transforms.functional.to_tensor(image).to(device)
image.sub_(mean[:, None, None]).div_(std[:, None, None])
return image[None, ...]

parse_objects = ParseObjects(topology)
draw_objects = DrawObjects(topology)

def execute(change):
data = preprocess(change)
cmap, paf = model_trt(data)
cmap, paf = cmap.detach().cpu(), paf.detach().cpu()
counts, objects, peaks = parse_objects(cmap, paf)
draw_objects(change, counts, objects, peaks)

def gstreamer_pipeline(
capture_width=1280,
capture_height=720,
display_width=1280,
display_height=720,
framerate=60,
flip_method=2,

):
return (
"nvarguscamerasrc ! "
"video/x-raw(memory:NVMM), "
"width=(int)%d, height=(int)%d, "
"format=(string)NV12, framerate=(fraction)%d/1 ! "
"nvvidconv flip-method=%d ! "
"video/x-raw, width=(int)%d, height=(int)%d, format=(string)BGRx ! "
"videoconvert ! "
"video/x-raw, format=(string)BGR ! appsink"
% (
capture_width,
capture_height,
framerate,
flip_method,
display_width,
display_height,))

def main():
capture_width = 1280
capture_height = 720
display_width = 1280
display_height = 720
framerate = 60
flip_method = 2
cam = cv2.VideoCapture(gstreamer_pipeline(flip_method=2), cv2.CAP_GSTREAMER)
while True:
ret, frame = cam.read()
frame = cv2.resize(frame,(224,224))
execute(frame)
frame = cv2.resize(frame,(640,640))
fps = cam.get(cv2.CAP_PROP_FPS)
cv2.putText(frame, 'FPS = '+str(fps), (50,50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 5)
cv2.imshow("test", frame)
if not ret:
break
key = cv2.waitKey(1)
# To close the window press "Q"
if key & 0xFF == ord('q') or key ==27:
break
cam.release()
cv2.destroyAllWindows()
if name == "main":
main() `

[SijinJohn]

@SarthakGarg19 @lbq779660843 i am getting error in this line ""frame = cv2.resize(frame,(224,224))"".
Why is that?
error:-src\resize.cpp:3720: error: (-215:Assertion failed) !ssize.empty() in function ‘cv2.resize’

[SarthakGarg19]

@SarthakGarg19 @lbq779660843 i am getting error in this line ""frame = cv2.resize(frame,(224,224))"".
Why is that?
error:-src\resize.cpp:3720: error: (-215:Assertion failed) !ssize.empty() in function ‘cv2.resize’

@SijinJohn I believe there might be some problem with your camera as the 'frame' object is empty and therefore is not able to resize, hence this error.

Please try running the following code and kindly confirm if the 'frame' is empty or not.

cam = cv2.VideoCapture(gstreamer_pipeline(flip_method=2), cv2.CAP_GSTREAMER)
while True:
ret, frame = cam.read()
print(frame)
type(frame)

Also refer to stack overflow : https://stackoverflow.com/questions/58242990/cv2-image-error-error-215assertion-failed-ssize-empty-in-function-cv

[SijinJohn]

def main():

cam = cv2.VideoCapture(0)

cv2.namedWindow("test")
cam.set(cv2.CAP_PROP_FRAME_WIDTH, WIDTH)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT, HEIGHT)

img_counter = 0

while True:
    ret, frame = cam.read()
    print(frame)
   #print("shape is ",frame.shape)
    #frame = cv2.resize(frame,(224,224))

    execute(frame)
    frame = cv2.resize(frame,(640,640))
    cv2.imshow("test", frame)
    if not ret:
        break
    key = cv2.waitKey(1)

    # To close the window press "Q"
    if key & 0xFF == ord('q') or key ==27:
        break


cam.release()

cv2.destroyAllWindows()

if name == "main":
main()

This gives error:-
None

error Traceback (most recent call last)
in
1 if name == "main":
----> 2 main()

in main()
15 # frame = cv2.resize(frame,(224,224))
16
---> 17 execute(frame)
18 frame = cv2.resize(frame,(640,640))
19 cv2.imshow("test", frame)

in execute(change)
1 def execute(change):
----> 2 data = preprocess(change)
3 cmap, paf = model_trt(data)
4 cmap, paf = cmap.detach().cpu(), paf.detach().cpu()
5 counts, objects, peaks = parse_objects(cmap, paf)

in preprocess(image)
2 global device
3 device = torch.device('cuda')
----> 4 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
5 image = PIL.Image.fromarray(image)
6 image = transforms.functional.to_tensor(image).to(device)

error: OpenCV(4.1.1) /home/nvidia/host/build_opencv/nv_opencv/modules/imgproc/src/color.cpp:182: error: (-215:Assertion failed) !_src.empty() in function 'cvtColor'

[SijinJohn]

here i am getting frame value as None.

[SijinJohn]

@SarthakGarg19 @lbq779660843 @kedaya1999

import cv2
import numpy as np

cam = cv2.VideoCapture(gstreamer_pipeline(flip_method=2), cv2.CAP_GSTREAMER)

cam = cv2.VideoCapture(0)
while True:
ret, frame = cam.read()

print(frame)

print("shape1is ",frame.shape)
frame = cv2.resize(frame,(640,480))
print("shape2is ",frame.shape)
print('type is')
type(frame)

cv2.imshow("Sijini",frame)

if cv2.waitKey(1)==13:
    cam.release()
    break

cam.release()
cv2.destroyAllWindows()

OUTPUT

shape1is (1080, 1920, 3)
shape2is (480, 640, 3)
type is
shape1is (1080, 1920, 3)
shape2is (480, 640, 3)
type is
shape1is (1080, 1920, 3)
shape2is (480, 640, 3)
type is

But here i am getting ootput for frame. Do you know why is this??

[SijinJohn]

ah it worked.
I removed these two line :-
cam.set(cv2.CAP_PROP_FRAME_WIDTH, WIDTH)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT, HEIGHT) and it worked.

Thanks

[SarthakGarg19]

I was going to suggest that! Glad you figured out on your own! 😃👍

[liubob-0]

Using OpenCV to open camera instead of jetcam because many a times jetcam faces issues with USB camera and fails to initialise the camera.
Also adding a python script named live_demo.py, which can be directly run for inference by typing "python3 live_demo.py"

nice work,but I tried it on jetson nano 2GB,got only 12FPS.I thought it could be 22FPS...

[SijinJohn]

@liubob-0 even i got 11-12 fps.

[GHao09]

Using OpenCV to open camera instead of jetcam because many a times jetcam faces issues with USB camera and fails to initialise the camera.
Also adding a python script named live_demo.py, which can be directly run for inference by typing "python3 live_demo.py"

nice work,but I tried it on jetson nano 2GB,got only 12FPS.I thought it could be 22FPS...
你好，我也是使用2GB的nano跑，请问怎么查看FPS呢？跑到12FPS你有对这个程序做什么调整吗？ @liubob-0

[GHao09]

@SijinJohn Hi,I ran this code on jetson nano 2GB,can you tell me how to check FPS?