Wednesday, June 30, 2021

The dazzling drone and light shows for the 100th anniversary of the CPC

NightHawkInLight

 

https://www.youtube.com/c/Nighthawkinlight/videos

kivy4 kivy language 1


#project structure
main.py
turorial.kv

#main.py
from kivy.uix.boxlayout import BoxLayout

class ScatterTextWidget(BoxLayout):
    pass

class TutorialApp(App):
    def build(self):
        return ScatterTextWidget()

TutorialApp().run()

-----------------
#tutorial.kv
<ScatterTextWidget>:
    orientation: 'vertical'
    TextInput:
        id: textinput_id
        font_size: 150
        size_hint_y: None
        height: 200
        text: 'default'
    FloatLayout:
        Scatter:
            Label:
                text: textinput_id.text
                font_size: 150

reference:

Sunday, June 27, 2021

kivy3 text binding

from kivy.app import App
from kivy.uix.button import Button
from kivy.uix.scatter import Scatter
from kivy.uix.label import Label
from kivy.uix.floatlayout import FloatLayout
from kivy.uix.textinput import TextInput
from kivy.uix.boxlayout import BoxLayout

class boxApp(App):
    def build(self):
        b = BoxLayout(orientation='vertical')
        t = TextInput(font_size=150, size_hint_y=None,
                      height=200, text='text')

        f = FloatLayout()
        s = Scatter()
        l = Label(text="Hello!", font_size=150)

        t.bind(text=l.setter('text'))

        f.add_widget(s)
        s.add_widget(l)

        b.add_widget(t)
        b.add_widget(f)
        return b

boxApp().run()

reference:

kivy2 box layout

from kivy.app import App
from kivy.uix.button import Button
from kivy.uix.scatter import Scatter
from kivy.uix.label import Label
from kivy.uix.floatlayout import FloatLayout
from kivy.uix.textinput import TextInput
from kivy.uix.boxlayout import BoxLayout

class buttonApp(App):
    def build(self):
        return Button(text = "Hello!",
                      background_color = (1, 0, 1, 1),
                      font_size=150)

class scatterApp(App):
    def build(self):
        f = FloatLayout()
        s = Scatter()
        l = Label(text="Hello!", font_size=150)

        f.add_widget(s)
        s.add_widget(l)
        return f

class boxApp(App):
    def build(self):
        b = BoxLayout()
        t = TextInput(font_size=150)

        f = FloatLayout()
        s = Scatter()
        l = Label(text="Hello!", font_size=150)

        f.add_widget(s)
        s.add_widget(l)

        b.add_widget(f)
        b.add_widget(t)
        return b

#buttonApp().run()
#scatterApp().run()
boxApp().run()

reference:

Friday, June 25, 2021

ubuntu python 2

 _ctypes
sudo apt install libffi-dev

install python 3.9
sudo apt update
sudo apt install software-properties-common
sudo add-apt-repository ppa:deadsnakes/ppa
sudo apt update
sudo apt install python3.9 python3.9-venv python3.9-dev

change python default version 3.9
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.9 2
sudo update-alternatives --install /usr/bin/python python /usr/bin/python2.7 3
sudo update-alternatives --config python

software -> download pycharm ->  setting -> interpreter -> venv interpreter python 3.9

 Cython
 pycharm terminal -> pip install Cython

python-ctypes
sudo apt-get update -y
sudo apt-get install -y python-ctypes
https://zoomadmin.com/HowToInstall/UbuntuPackage/python-ctypes

distutils.util
sudo apt-get install python3-distutils
sudo apt-get install python3-apt
https://askubuntu.com/questions/1239829/modulenotfounderror-no-module-named-distutils-util

buildozer
sudo pip install buildozer

buildozer init

zlib headers
sudo apt-get install zlib1g-dev

git
sudo apt install git


javac
sudo apt-get install openjdk-8-jdk

make
sudo apt install make

autoreconf
sudo apt-get install autoconf
sudo apt-get install dh-autoreconf

buildozer android debug

Monday, June 21, 2021

kivy 1

float layout scatter

button with purple background
#tutorial1.py
from kivy.app import App
from kivy.uix.button import Button
from kivy.uix.scatter import Scatter
from kivy.uix.label import Label
from kivy.uix.floatlayout import FloatLayout

class buttonApp(App):
    def build(self):
        return Button(text = "Hello!",
                      background_color = (1, 0, 1, 1),
                      font_size=150)

class scatterApp(App):
    def build(self):
        f = FloatLayout()
        s = Scatter()
        l = Label(text="Hello!", font_size=150)

        f.add_widget(s)
        s.add_widget(l)
        return f

#buttonApp().run()
scatterApp().run()

reference:

特高压输电

Saturday, June 19, 2021

opencv 66 seamless clone

elon mask is seamlessly cloned on mars surface


mask

#seamlessClone.py
import cv2
import numpy as np

# Read images : src image will be cloned into dst
dst = cv2.imread("assets/mars.jpg")
src = cv2.imread("assets/elon mask.jpg")

print(dst.shape, src.shape, src.dtype, dst.dtype)

# Create an all white mask
mask = src.copy()

mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)

#set white pixel to black, and the rest to white
mask = np.where(mask == 255, 0, 255).astype("uint8")
mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)

#remove salt and pepper noise
mask = cv2.medianBlur(mask, 15)

#mask dilation
kernal = np.ones((5, 5), np.uint8)
mask = cv2.dilate(mask, kernal, iterations=1)

cv2.imshow("mask", mask)

# The location of the center of the src in the dst
width, height, channels = dst.shape
center = (int(height / 2) - 200, int(width / 2))

# Seamlessly clone src into dst and put the results in output
normal_clone = cv2.seamlessClone(src, dst, mask, center, cv2.NORMAL_CLONE)
mixed_clone = cv2.seamlessClone(src, dst, mask, center, cv2.MIXED_CLONE)

cv2.imshow("seamless clone", mixed_clone)
cv2.waitKey(0)

reference:

Deepfake deep face lab

Friday, June 18, 2021

解密合肥发展密码

What fungi teach us

opencv 65 face mesh mediapipe


#face_mesh.py
import os
import cv2
import mediapipe as mp

cap = cv2.VideoCapture("assets/face mesh.mp4")

# Define the codec and create VideoWriter object
fps = 25.175
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
size = (int(frame_width), int(frame_height))
fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
path = 'C:/Users/zchen/PycharmProjects/opencv/googleNet/record'
out = cv2.VideoWriter()
success = out.open(os.path.join(path, "face mesh mediapipe.mov"), fourcc, fps, size, True)

mpFaceMesh = mp.solutions.face_mesh
faceMesh = mpFaceMesh.FaceMesh(max_num_faces=3)
mpDraw = mp.solutions.drawing_utils
drawSpec = mpDraw.DrawingSpec(thickness=1, circle_radius=1, color=(0, 0, 255))

while True:
    ret, frame = cap.read()

    if ret == False: break

    frame_RGB = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = faceMesh.process(frame_RGB)

    if results.multi_face_landmarks:
        for faceLms in results.multi_face_landmarks:
            for id, lm in enumerate(faceLms.landmark):
                #print(id, lm)
                h, w, c = frame.shape
                cx, cy = int(lm.x * w), int(lm.y * h)
                #print(id, cx, cy)

            mpDraw.draw_landmarks(frame, faceLms, mpFaceMesh.FACE_CONNECTIONS,
                                  drawSpec, drawSpec)

    cv2.imshow("frame", frame)
    out.write(frame)

    if cv2.waitKey(1) == ord('q'):
        break

    if cv2.waitKey(1) == ord('p'):
        cv2.waitKey(-1)  # wait until any key is pressed


cap.release()
cv2.destroyAllWindows()

reference:

Miami Vacations

Slime Obsidian


Wednesday, June 16, 2021

opencv 64 finger count


#finger_count.py
import os
import cv2
import mediapipe as mp
import time
import numpy as np
import math

cap = cv2.VideoCapture("assets/finger.mp4")

# Define the codec and create VideoWriter object
fps = 25.175
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
size = (int(frame_width), int(frame_height))
#print(size)
fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
path = 'C:/Users/zchen/PycharmProjects/opencv/googleNet/record'
out = cv2.VideoWriter()
success = out.open(os.path.join(path, "finger count.mov"), fourcc, fps, size, True)
mpHands = mp.solutions.hands
hands = mpHands.Hands()
mpDraw = mp.solutions.drawing_utils

index_up, middle_up, ring_up, pinky_up, thumb_up = False, False, False, False, False

while True:
    ret, frame = cap.read()

    if ret == False: break

    frame_RGB = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = hands.process(frame_RGB)

    if results.multi_hand_landmarks:
        for handLms in results.multi_hand_landmarks:
            """
            for id, lm in enumerate(handLms.landmark):
                #print(id, lm)
                h, w, c = frame.shape
                cx, cy = int(lm.x * w), int(lm.y * h)
                #print(id, cx, cy)
                #draw circle at thumb tip
                if id == 4:
                    cv2.circle(frame, (cx, cy), 15, (255, 0, 255), cv2.FILLED)
            """
            h_marks = handLms.landmark
            #print(h_marks[4], h_marks[8])

            index_up = h_marks[8].y < h_marks[7].y
            middle_up = h_marks[12].y < h_marks[11].y
            ring_up = h_marks[16].y < h_marks[15].y
            pinky_up = h_marks[20].y < h_marks[19].y
            thumb_up = h_marks[4].x < h_marks[3].x

            print(index_up, middle_up, ring_up, pinky_up, thumb_up)

            mpDraw.draw_landmarks(frame, handLms, mpHands.HAND_CONNECTIONS)

    if thumb_up:
        cv2.line(frame, (200, 380), (130, 200), (255, 0, 0), 30)
    if index_up:
        cv2.line(frame, (220, 280), (220, 50), (255, 0, 0), 23)
    if middle_up:
        cv2.line(frame, (270, 280), (270, 30), (255, 0, 0), 25)
    if ring_up:
        cv2.line(frame, (320, 200), (320, 40), (255, 0, 0), 22)
    if pinky_up:
        cv2.line(frame, (370, 280), (370, 90), (255, 0, 0), 15)
    cv2.rectangle(frame, (200, 200), (380, 400), (255, 0, 0), cv2.FILLED)

    cv2.imshow("frame", frame)
    out.write(frame)

    if cv2.waitKey(1) == ord('q'):
        break

    if cv2.waitKey(1) == ord('p'):
        cv2.waitKey(-1)  # wait until any key is pressed

cap.release()
cv2.destroyAllWindows()

reference:

曹县汉服

Monday, June 14, 2021

opencv 63 gesture volume


#guesture.py
import os
import cv2
import mediapipe as mp
import time
import numpy as np
import math

cap = cv2.VideoCapture("assets/guesture.mp4")

# Define the codec and create VideoWriter object
fps = 25.175
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
size = (int(frame_width), int(frame_height))
print(size)
fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
path = 'C:/Users/zchen/PycharmProjects/opencv/googleNet/record'
out = cv2.VideoWriter()
success = out.open(os.path.join(path, "guesture volume.mov"), fourcc, fps, size, True)
mpHands = mp.solutions.hands
hands = mpHands.Hands()
mpDraw = mp.solutions.drawing_utils

pTime = 0

while True:
    ret, frame = cap.read()

    if ret == False: break
    cTime = time.time()
    fps = 1/(cTime - pTime)
    pTime = cTime

    frame_RGB = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = hands.process(frame_RGB)

    cv2.putText(frame, f'FPS: {int(fps)}', (100, 100), cv2.FONT_HERSHEY_COMPLEX, 3,
                                            (255, 0, 0), 3)

    h, w, c = frame.shape
    vol = 0

    if results.multi_hand_landmarks:
        for handLms in results.multi_hand_landmarks:
            """
            for id, lm in enumerate(handLms.landmark):
                #print(id, lm)
                h, w, c = frame.shape
                cx, cy = int(lm.x * w), int(lm.y * h)
                #print(id, cx, cy)
                #draw circle at thumb tip
                if id == 4:
                    cv2.circle(frame, (cx, cy), 15, (255, 0, 255), cv2.FILLED)
            """
            h_marks = handLms.landmark
            #print(h_marks[4], h_marks[8])

            x1, y1 = int(h_marks[4].x * w), int(h_marks[4].y * h)
            x2, y2 = int(h_marks[8].x * w), int(h_marks[8].y * h)
            cx, cy = (x1+x2)//2, (y1+y2)//2
            #print(x1, y1)
            cv2.circle(frame, (x1, y1), 30, (255, 0, 255), cv2.FILLED)
            cv2.circle(frame, (x2, y2), 30, (255, 0, 255), cv2.FILLED)
            cv2.line(frame, (x1, y1), (x2, y2), (255, 0, 255), 3)
            cv2.circle(frame, (cx, cy), 30, (255, 0, 255), cv2.FILLED)

            length = math.hypot(x2 - x1, y2 - y1)
            #print(length)
            if length < 100:
                cv2.circle(frame, (cx, cy), 30, (0, 255, 0), cv2.FILLED)

            vol = np.interp(length, [100, 800], [100, 1000])

            mpDraw.draw_landmarks(frame, handLms, mpHands.HAND_CONNECTIONS)

        cv2.rectangle(frame, (100, 200), (1000, 300), (255, 0, 0), 3)
        cv2.rectangle(frame, (100, 200), (int(vol), 300), (255, 0, 0), cv2.FILLED)

        cv2.putText(frame, f'Vol: {int((vol-100)//9)}%', (600, 100), cv2.FONT_HERSHEY_COMPLEX, 3,
                    (255, 0, 0), 3)

    cv2.imshow("frame", frame)
    out.write(frame)

    if cv2.waitKey(1) == ord('q'):
        break

    if cv2.waitKey(1) == ord('p'):
        cv2.waitKey(-1)  # wait until any key is pressed

cap.release()
cv2.destroyAllWindows()

reference:

hand detection

Saturday, June 12, 2021

opencv 62 hand detection mediapipe

25 fps
#hand.py
import os
import cv2
import mediapipe as mp
import time

cap = cv2.VideoCapture("assets/sign_language.mp4")

# Define the codec and create VideoWriter object
fps = 25.175
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
size = (int(frame_width), int(frame_height))
fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
path = 'C:/Users/zchen/PycharmProjects/opencv/googleNet/record'
out = cv2.VideoWriter()
success = out.open(os.path.join(path, "hand tracking mediapipe.mov"), fourcc, fps, size, True)

frame_count = 0
t1 = time.time()

mpHands = mp.solutions.hands
hands = mpHands.Hands()
mpDraw = mp.solutions.drawing_utils

while True:
    ret, frame = cap.read()

    if ret == False: break

    frame_RGB = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = hands.process(frame_RGB)

    if results.multi_hand_landmarks:
        for handLms in results.multi_hand_landmarks:
            for id, lm in enumerate(handLms.landmark):
                #print(id, lm)
                h, w, c = frame.shape
                cx, cy = int(lm.x * w), int(lm.y * h)
                #print(id, cx, cy)
                #draw circle at thumb tip
                if id == 4:
                    cv2.circle(frame, (cx, cy), 15, (255, 0, 255), cv2.FILLED)

            mpDraw.draw_landmarks(frame, handLms, mpHands.HAND_CONNECTIONS)

    cv2.imshow("frame", frame)
    out.write(frame)
    frame_count += 1

    if cv2.waitKey(1) == ord('q'):
        break

    if cv2.waitKey(1) == ord('p'):
        cv2.waitKey(-1)  # wait until any key is pressed

t2 = time.time()
# calculate FPS
fps = str(float(frame_count / float(t2 - t1))) + ' FPS'
print("Frames processed: {}".format(frame_count))
print("Elapsed time: {:.2f}".format(float(t2 - t1)))
print("FPS: {}".format(fps))

cap.release()
cv2.destroyAllWindows()

#reference:

Thursday, June 10, 2021

opencv 61 openpose multi person detection


#pose_multi.py
import os
import cv2
import time
import numpy as np
import matplotlib.pyplot as plt
# %matplotlib inline
import matplotlib

from random import randint

protoFile = "googleNet/body_18/pose_deploy_linevec.prototxt"
weightsFile = "googleNet/body_18/pose_iter_440000.caffemodel"
net = cv2.dnn.readNetFromCaffe(protoFile, weightsFile)

# set CUDA as the preferable backend and target
print("[INFO] setting preferable backend and target to CUDA...")
net.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA)
net.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA)

nPoints = 18
# COCO Output Format
keypointsMapping = ['Nose', 'Neck', 'R-Sho', 'R-Elb', 'R-Wr', 'L-Sho',
                    'L-Elb', 'L-Wr', 'R-Hip', 'R-Knee', 'R-Ank', 'L-Hip',
                    'L-Knee', 'L-Ank', 'R-Eye', 'L-Eye', 'R-Ear', 'L-Ear']

POSE_PAIRS = [[1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7],
              [1, 8], [8, 9], [9, 10], [1, 11], [11, 12], [12, 13],
              [1, 0], [0, 14], [14, 16], [0, 15], [15, 17],
              [2, 17], [5, 16]]

# index of pafs correspoding to the POSE_PAIRS
# e.g for POSE_PAIR(1,2), the PAFs are located at indices (31,32) of output, Similarly, (1,5) -> (39,40) and so on.
mapIdx = [[31, 32], [39, 40], [33, 34], [35, 36], [41, 42], [43, 44],
          [19, 20], [21, 22], [23, 24], [25, 26], [27, 28], [29, 30],
          [47, 48], [49, 50], [53, 54], [51, 52], [55, 56],
          [37, 38], [45, 46]]

colors = [[0, 100, 255], [0, 100, 255], [0, 255, 255], [0, 100, 255], [0, 255, 255], [0, 100, 255],
          [0, 255, 0], [255, 200, 100], [255, 0, 255], [0, 255, 0], [255, 200, 100], [255, 0, 255],
          [0, 0, 255], [255, 0, 0], [200, 200, 0], [255, 0, 0], [200, 200, 0], [0, 0, 0]]


# Find the Keypoints using Non Maximum Suppression on the Confidence Map
def getKeypoints(probMap, threshold=0.1):
    mapSmooth = cv2.GaussianBlur(probMap, (3, 3), 0, 0)

    mapMask = np.uint8(mapSmooth > threshold)
    keypoints = []

    # find the blobs
    contours, _ = cv2.findContours(mapMask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

    # for each blob find the maxima
    for cnt in contours:
        blobMask = np.zeros(mapMask.shape)
        blobMask = cv2.fillConvexPoly(blobMask, cnt, 1)
        maskedProbMap = mapSmooth * blobMask
        _, maxVal, _, maxLoc = cv2.minMaxLoc(maskedProbMap)
        keypoints.append(maxLoc + (probMap[maxLoc[1], maxLoc[0]],))

    return keypoints


# Find valid connections between the different joints of a all persons present
def getValidPairs(output):
    valid_pairs = []
    invalid_pairs = []
    n_interp_samples = 10
    paf_score_th = 0.1
    conf_th = 0.7
    # loop for every POSE_PAIR
    for k in range(len(mapIdx)):
        # A->B constitute a limb
        pafA = output[0, mapIdx[k][0], :, :]
        pafB = output[0, mapIdx[k][1], :, :]
        pafA = cv2.resize(pafA, (frameWidth, frameHeight))
        pafB = cv2.resize(pafB, (frameWidth, frameHeight))

        # Find the keypoints for the first and second limb
        candA = detected_keypoints[POSE_PAIRS[k][0]]
        candB = detected_keypoints[POSE_PAIRS[k][1]]
        nA = len(candA)
        nB = len(candB)

        # If keypoints for the joint-pair is detected
        # check every joint in candA with every joint in candB
        # Calculate the distance vector between the two joints
        # Find the PAF values at a set of interpolated points between the joints
        # Use the above formula to compute a score to mark the connection valid

        if (nA != 0 and nB != 0):
            valid_pair = np.zeros((0, 3))
            for i in range(nA):
                max_j = -1
                maxScore = -1
                found = 0
                for j in range(nB):
                    # Find d_ij
                    d_ij = np.subtract(candB[j][:2], candA[i][:2])
                    norm = np.linalg.norm(d_ij)
                    if norm:
                        d_ij = d_ij / norm
                    else:
                        continue
                    # Find p(u)
                    interp_coord = list(zip(np.linspace(candA[i][0], candB[j][0], num=n_interp_samples),
                                            np.linspace(candA[i][1], candB[j][1], num=n_interp_samples)))
                    # Find L(p(u))
                    paf_interp = []
                    for k in range(len(interp_coord)):
                        paf_interp.append([pafA[int(round(interp_coord[k][1])), int(round(interp_coord[k][0]))],
                                           pafB[int(round(interp_coord[k][1])), int(round(interp_coord[k][0]))]])
                        # Find E
                    paf_scores = np.dot(paf_interp, d_ij)
                    avg_paf_score = sum(paf_scores) / len(paf_scores)

                    # Check if the connection is valid
                    # If the fraction of interpolated vectors aligned with PAF is higher then threshold -> Valid Pair
                    if (len(np.where(paf_scores > paf_score_th)[0]) / n_interp_samples) > conf_th:
                        if avg_paf_score > maxScore:
                            max_j = j
                            maxScore = avg_paf_score
                            found = 1
                # Append the connection to the list
                if found:
                    valid_pair = np.append(valid_pair, [[candA[i][3], candB[max_j][3], maxScore]], axis=0)

            # Append the detected connections to the global list
            valid_pairs.append(valid_pair)
        else:  # If no keypoints are detected
            #             print("No Connection : k = {}".format(k))
            invalid_pairs.append(k)
            valid_pairs.append([])
    #     print(valid_pairs)
    return valid_pairs, invalid_pairs


# This function creates a list of keypoints belonging to each person
# For each detected valid pair, it assigns the joint(s) to a person
# It finds the person and index at which the joint should be added. This can be done since we have an id for each joint
def getPersonwiseKeypoints(valid_pairs, invalid_pairs):
    # the last number in each row is the overall score
    personwiseKeypoints = -1 * np.ones((0, 19))

    for k in range(len(mapIdx)):
        if k not in invalid_pairs:
            partAs = valid_pairs[k][:, 0]
            partBs = valid_pairs[k][:, 1]
            indexA, indexB = np.array(POSE_PAIRS[k])

            for i in range(len(valid_pairs[k])):
                found = 0
                person_idx = -1
                for j in range(len(personwiseKeypoints)):
                    if personwiseKeypoints[j][indexA] == partAs[i]:
                        person_idx = j
                        found = 1
                        break

                if found:
                    personwiseKeypoints[person_idx][indexB] = partBs[i]
                    personwiseKeypoints[person_idx][-1] += keypoints_list[partBs[i].astype(int), 2] + valid_pairs[k][i][
                        2]

                # if find no partA in the subset, create a new subset
                elif not found and k < 17:
                    row = -1 * np.ones(19)
                    row[indexA] = partAs[i]
                    row[indexB] = partBs[i]
                    # add the keypoint_scores for the two keypoints and the paf_score
                    row[-1] = sum(keypoints_list[valid_pairs[k][i, :2].astype(int), 2]) + valid_pairs[k][i][2]
                    personwiseKeypoints = np.vstack([personwiseKeypoints, row])
    return personwiseKeypoints


# capture video
video_path = "assets/mma.mp4"
cap = cv2.VideoCapture(video_path)
# Check if video file is opened successfully
if (cap.isOpened() == False):
    print("Error opening video stream or file")

# We need to set resolutions.
# so, convert them from float to integer.
frame_width = int(cap.get(3))
frame_height = int(cap.get(4))
size = (frame_width, frame_height)
# result = cv2.VideoWriter('pose.avi',
#                          cv2.VideoWriter_fourcc(*'MJPG'),
#                          15, size)
# Define the codec and create VideoWriter object
fps = 5
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
size = (int(frame_width), int(frame_height))
fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
path = 'C:/Users/zchen/PycharmProjects/opencv/googleNet/record'
out = cv2.VideoWriter()
success = out.open(os.path.join(path, "mma.mov"), fourcc, fps, size, True)

total = 0
count = 0
while (cap.isOpened()):
    # Capture frame-by-frame
    ret, frame = cap.read()
    start = time.time()
    frameWidth = frame.shape[1]
    frameHeight = frame.shape[0]
    if ret == True:
        inHeight = 368
        inWidth = int((inHeight / frameHeight) * frameWidth)
        #         frame = cv2.resize(frame, (inHeight, inWidth), cv2.INTER_AREA)
        # process the frame here
        inpBlob = cv2.dnn.blobFromImage(frame, 1.0 / 255, (inWidth, inHeight), (0, 0, 0), swapRB=False, crop=False)
        net.setInput(inpBlob)
        output = net.forward()
        #         print(type(output))
        detected_keypoints = []
        keypoints_list = np.zeros((0, 3))
        keypoint_id = 0
        threshold = 0.1

        for part in range(nPoints):
            probMap = output[0, part, :, :]
            probMap = cv2.resize(probMap, (frame.shape[1], frame.shape[0]))
            #     plt.figure()
            #     plt.imshow(255*np.uint8(probMap>threshold))
            keypoints = getKeypoints(probMap, threshold)
            print("Keypoints - {} : {}".format(keypointsMapping[part], keypoints))
            keypoints_with_id = []
            for i in range(len(keypoints)):
                keypoints_with_id.append(keypoints[i] + (keypoint_id,))
                keypoints_list = np.vstack([keypoints_list, keypoints[i]])
                keypoint_id += 1

            detected_keypoints.append(keypoints_with_id)
        #         frameClone = frame.copy()

        for i in range(nPoints):
            for j in range(len(detected_keypoints[i])):
                cv2.circle(frame, detected_keypoints[i][j][0:2], 5, [0, 255, 255], -1, cv2.LINE_AA)
        #         plt.figure(figsize=[15,15])
        #         plt.imshow(frameClone[:,:,[2,1,0]])
        valid_pairs, invalid_pairs = getValidPairs(output)
        personwiseKeypoints = getPersonwiseKeypoints(valid_pairs, invalid_pairs)
        for i in range(17):
            for n in range(len(personwiseKeypoints)):
                index = personwiseKeypoints[n][np.array(POSE_PAIRS[i])]
                if -1 in index:
                    continue
                B = np.int32(keypoints_list[index.astype(int), 0])
                A = np.int32(keypoints_list[index.astype(int), 1])
                cv2.line(frame, (B[0], A[0]), (B[1], A[1]), colors[i], 3, cv2.LINE_AA)

        #         plt.figure(figsize=[15,15])
        #         plt.imshow(frameClone[:,:,[2,1,0]])
        cv2.imshow('frameclone', frame)
        out.write(frame)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
        count += 1
        total += (time.time() - start)
        print(count, count / total, (time.time() - start))

    # Break the loop
    else:
        break

out.release()
cap.release()
cv2.destroyAllWindows()

-----------------------
#logs
Keypoints - Nose : [(1065, 528, 0.6999403)]
Keypoints - Neck : [(1136, 504, 0.63968205), (877, 388, 0.7440788)]
Keypoints - R-Sho : [(1158, 481, 0.45033738), (971, 387, 0.75994366)]
Keypoints - R-Elb : [(994, 529, 0.7064167)]
Keypoints - R-Wr : [(971, 645, 0.7172611)]
Keypoints - L-Sho : [(1135, 527, 0.74793553), (783, 411, 0.6718034)]
Keypoints - L-Elb : [(1160, 622, 0.64027196), (713, 575, 0.6784791)]
Keypoints - L-Wr : [(1135, 692, 0.46578476), (621, 692, 0.7087032)]
Keypoints - R-Hip : [(924, 693, 0.53040093), (1323, 552, 0.28071627)]
Keypoints - R-Knee : [(1275, 738, 0.21219043)]
Keypoints - R-Ank : []
Keypoints - L-Hip : [(831, 715, 0.5052447), (1323, 574, 0.5223487)]
Keypoints - L-Knee : [(854, 951, 0.56000316), (1392, 716, 0.7037562)]
Keypoints - L-Ank : [(1580, 739, 0.39099985)]
Keypoints - R-Eye : []
Keypoints - L-Eye : [(1065, 527, 0.7943738)]
Keypoints - R-Ear : [(901, 246, 0.8871688)]
Keypoints - L-Ear : [(1088, 504, 0.9218901), (784, 246, 0.9275103)]

reference:

model download

Part Affinity Field (PAF) algorithm explaination

Tuesday, June 8, 2021

opencv 60 openpose heatmap


heat map of neck

nose

right hip

right knee

right big toe
#project structure
pose_multi.py
googleNet
    body_25
        pose_deploy.prototxt
        pose_iter_584000.caffemodel

#pose_multi.py
import cv2
import numpy as np

protoFile = "googleNet/body_25/pose_deploy.prototxt"
weightsFile = "googleNet/body_25/pose_iter_584000.caffemodel"
net = cv2.dnn.readNetFromCaffe(protoFile, weightsFile)

image1 = cv2.imread("assets/people_group.jpg")
frameHeight, frameWidth, channel = image1.shape

# Fix the input Height and get the width according to the Aspect Ratio
inHeight = 368
inWidth = int((inHeight / frameHeight) * frameWidth)

inpBlob = cv2.dnn.blobFromImage(image1, 1.0 / 255, (inWidth, inHeight),
                                (0, 0, 0), swapRB=False, crop=False)

net.setInput(inpBlob)
output = net.forward()

#probMap shows probility of each pixel being a part of nose
#i = 0, 0 represent nose in trained model, 1 represent neck...
"""
// Result for BODY_25 (25 body parts consisting of COCO + foot)
// const std::map<unsigned int, std::string> POSE_BODY_25_BODY_PARTS {
//     {0,  "Nose"},
//     {1,  "Neck"},
//     {2,  "RShoulder"},
//     {3,  "RElbow"},
//     {4,  "RWrist"},
//     {5,  "LShoulder"},
//     {6,  "LElbow"},
//     {7,  "LWrist"},
//     {8,  "MidHip"},
//     {9,  "RHip"},
//     {10, "RKnee"},
//     {11, "RAnkle"},
//     {12, "LHip"},
//     {13, "LKnee"},
//     {14, "LAnkle"},
//     {15, "REye"},
//     {16, "LEye"},
//     {17, "REar"},
//     {18, "LEar"},
//     {19, "LBigToe"},
//     {20, "LSmallToe"},
//     {21, "LHeel"},
//     {22, "RBigToe"},
//     {23, "RSmallToe"},
//     {24, "RHeel"},
//     {25, "Background"}
// };
"""

i = 22
probMap = output[0, i, :, :]
probMap = cv2.resize(probMap, (frameWidth, frameHeight))
probMap_RGB = cv2.cvtColor(probMap, cv2.COLOR_GRAY2RGB)

#probablity is between 0 and 1, multiply 255 to show heat map
probMap_RGB = np.multiply(probMap_RGB, 255).astype("uint8")
#print(probMap_RGB.shape, image1.shape)
#print(probMap_RGB, image1)

blend = cv2.addWeighted(image1.astype("uint8"), 0.5, probMap_RGB, 0.5, 0)

cv2.imshow("probMap", blend)
cv2.waitKey(0)

reference:

download model

why rich country has less working hours

Monday, June 7, 2021

Sunday, June 6, 2021

opencv 59 resnet person detection + pose estimation


persons are detect by resnet
pose estimations are applied to different person


#pose_tfod.py
import os
import time
import tensorflow as tf
import tarfile
import urllib.request
from object_detection.utils import label_map_util
from object_detection.utils import config_util
from object_detection.utils import visualization_utils as viz_utils
from object_detection.builders import model_builder
import cv2
import numpy as np

DATA_DIR = os.path.join(os.getcwd(), 'googleNet')
MODELS_DIR = os.path.join(DATA_DIR, 'modelZoo')
print(MODELS_DIR)
for dir in [DATA_DIR, MODELS_DIR]:
    if not os.path.exists(dir):
        os.mkdir(dir)

# Download and extract model
MODEL_DATE = '20200711'
MODEL_NAME = 'centernet_resnet101_v1_fpn_512x512_coco17_tpu-8'
MODEL_TAR_FILENAME = MODEL_NAME + '.tar.gz'
MODELS_DOWNLOAD_BASE = 'http://download.tensorflow.org/models/object_detection/tf2/'
MODEL_DOWNLOAD_LINK = MODELS_DOWNLOAD_BASE + MODEL_DATE + '/' + MODEL_TAR_FILENAME
PATH_TO_MODEL_TAR = os.path.join(MODELS_DIR, MODEL_TAR_FILENAME)
PATH_TO_CKPT = os.path.join(MODELS_DIR, os.path.join(MODEL_NAME, 'checkpoint/'))
PATH_TO_CFG = os.path.join(MODELS_DIR, os.path.join(MODEL_NAME, 'pipeline.config'))
if not os.path.exists(PATH_TO_CKPT):
    print('Downloading model. This may take a while... ', end='')
    urllib.request.urlretrieve(MODEL_DOWNLOAD_LINK, PATH_TO_MODEL_TAR)
    tar_file = tarfile.open(PATH_TO_MODEL_TAR)
    tar_file.extractall(MODELS_DIR)
    tar_file.close()
    os.remove(PATH_TO_MODEL_TAR)
    print('Done')

# Download labels file
LABEL_FILENAME = 'mscoco_label_map.pbtxt'
LABELS_DOWNLOAD_BASE = \
    'https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/'
PATH_TO_LABELS = os.path.join(MODELS_DIR, os.path.join(MODEL_NAME, LABEL_FILENAME))
if not os.path.exists(PATH_TO_LABELS):
    print('Downloading label file... ', end='')
    print(PATH_TO_LABELS)
    urllib.request.urlretrieve(LABELS_DOWNLOAD_BASE + LABEL_FILENAME, PATH_TO_LABELS)
    print('Done')

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'  # Suppress TensorFlow logging
tf.get_logger().setLevel('ERROR')  # Suppress TensorFlow logging (2)

# Enable GPU dynamic memory allocation
gpus = tf.config.experimental.list_physical_devices('GPU')
for gpu in gpus:
    tf.config.experimental.set_memory_growth(gpu, True)

# Load pipeline config and build a detection model
configs = config_util.get_configs_from_pipeline_file(PATH_TO_CFG)
model_config = configs['model']
detection_model = model_builder.build(model_config=model_config, is_training=False)

# Restore checkpoint
ckpt = tf.compat.v2.train.Checkpoint(model=detection_model)
ckpt.restore(os.path.join(PATH_TO_CKPT, 'ckpt-0')).expect_partial()


@tf.function
def detect_fn(image):
    """Detect objects in image."""

    image, shapes = detection_model.preprocess(image)
    prediction_dict = detection_model.predict(image, shapes)
    detections = detection_model.postprocess(prediction_dict, shapes)

    return detections, prediction_dict, tf.reshape(shapes, [-1])


category_index = label_map_util.create_category_index_from_labelmap(PATH_TO_LABELS,
                                                                    use_display_name=True)
#print(category_index)

BODY_PARTS = {"Nose": 0, "Neck": 1, "RShoulder": 2, "RElbow": 3, "RWrist": 4,
              "LShoulder": 5, "LElbow": 6, "LWrist": 7, "RHip": 8, "RKnee": 9,
              "RAnkle": 10, "LHip": 11, "LKnee": 12, "LAnkle": 13, "REye": 14,
              "LEye": 15, "REar": 16, "LEar": 17, "Background": 18}

POSE_PAIRS = [["Neck", "RShoulder"], ["Neck", "LShoulder"], ["RShoulder", "RElbow"],
              ["RElbow", "RWrist"], ["LShoulder", "LElbow"], ["LElbow", "LWrist"],
              ["Neck", "RHip"], ["RHip", "RKnee"], ["RKnee", "RAnkle"], ["Neck", "LHip"],
              ["LHip", "LKnee"], ["LKnee", "LAnkle"], ["Neck", "Nose"], ["Nose", "REye"],
              ["REye", "REar"], ["Nose", "LEye"], ["LEye", "LEar"]]

inWidth = 368
inHeight = 368

net = cv2.dnn.readNetFromTensorflow("googleNet/graph_opt.pb")

# Using OpenCV to initialize the webcam
cap = cv2.VideoCapture("assets/mma.mp4")

# Define the codec and create VideoWriter object
fps = 20
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
size = (int(frame_width), int(frame_height))
fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
path = 'C:/Users/zchen/PycharmProjects/opencv/googleNet/record'
out = cv2.VideoWriter()
success = out.open(os.path.join(path, "mma.mov"), fourcc, fps, size, True)

frame_count = 0
t1 = time.time()

while cap.isOpened():
    ret, image_np = cap.read()
    input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32)
    detections, predictions_dict, shapes = detect_fn(input_tensor)

    label_id_offset = 1
    image_np_with_detections = image_np.copy()

    """
    viz_utils.visualize_boxes_and_labels_on_image_array(
        image_np_with_detections,
        detections['detection_boxes'][0].numpy(),
        (detections['detection_classes'][0].numpy() + label_id_offset).astype(int),
        detections['detection_scores'][0].numpy(),
        category_index,
        use_normalized_coordinates=True,
        max_boxes_to_draw=200,
        min_score_thresh=.50,
        line_thickness=1,
        agnostic_mode=False)
    """

    categories = (detections['detection_classes'][0].numpy() + label_id_offset).astype(int)
    boxes = detections['detection_boxes'][0].numpy()
    scores = detections['detection_scores'][0].numpy()
    #print(categories, boxes, scores)

    for i, category in enumerate(categories):
        if category == 1 and scores[i] > 0.5:
            start_x = int(frame_width * boxes[i][1])
            start_y = int(frame_height * boxes[i][0])
            end_x = int(frame_width * boxes[i][3])
            end_y = int(frame_height * boxes[i][2])
            #print(start_x, start_y, end_x, end_y)

            """
            cv2.rectangle(image_np_with_detections,
                          (start_x, start_y),
                          (end_x, end_y),
                          (0, 255, 255), 1)
            """

            person = image_np_with_detections[start_y:end_y, start_x:end_x]
            personHeight, personWidth, channels = person.shape
            #print(person.shape)

            if personWidth > 0 and personHeight > 0:

                net.setInput(
                    cv2.dnn.blobFromImage(person, 1.0, (inWidth, inHeight), (127.5, 127.5, 127.5), swapRB=True, crop=False))
                pose_out = net.forward()
                pose_out = pose_out[:, :19, :, :]  # MobileNet output [1, 57, -1, -1], we only need the first 19 elements

                assert (len(BODY_PARTS) == pose_out.shape[1])

                points = []
                for j in range(len(BODY_PARTS)):
                    # Slice heatmap of corresponging body's part.
                    heatMap = pose_out[0, j, :, :]

                    # Originally, we try to find all the local maximums. To simplify a sample
                    # we just find a global one. However only a single pose at the same time
                    # could be detected this way.
                    _, conf, _, point = cv2.minMaxLoc(heatMap)
                    x = (personWidth * point[0]) / pose_out.shape[3]
                    y = (personHeight * point[1]) / pose_out.shape[2]
                    # Add a point if it's confidence is higher than threshold.
                    points.append((int(x), int(y)) if conf > 0.2 else None)

                for pair in POSE_PAIRS:
                    partFrom = pair[0]
                    partTo = pair[1]
                    assert (partFrom in BODY_PARTS)
                    assert (partTo in BODY_PARTS)

                    idFrom = BODY_PARTS[partFrom]
                    idTo = BODY_PARTS[partTo]

                    if points[idFrom] and points[idTo]:
                        cv2.line(person, points[idFrom], points[idTo], (0, 255, 0), 3)
                        cv2.ellipse( person, points[idFrom], (3, 3), 0, 0, 360, (0, 0, 255), cv2.FILLED)
                        cv2.ellipse( person, points[idTo], (3, 3), 0, 0, 360, (0, 0, 255), cv2.FILLED)

                #cv2.imshow("person", person)

                if cv2.waitKey(1) == ord('p'):
                    cv2.waitKey(-1)  # wait until any key is pressed

    cv2.imshow('centernet_resnet101_pose_detection', image_np_with_detections)
    out.write(image_np_with_detections)
    frame_count += 1

    if cv2.waitKey(1) == ord('q'):
        break

    if cv2.waitKey(1) == ord('p'):
        cv2.waitKey(-1)  # wait until any key is pressed

t2 = time.time()
# calculate FPS
fps = str(float(frame_count / float(t2 - t1))) + ' FPS'
print("Frames processed: {}".format(frame_count))
print("Elapsed time: {:.2f}".format(float(t2 - t1)))
print("FPS: {}".format(fps))

# Release camera and close windows
cap.release()
cv2.destroyAllWindows()

reference:
resnet

pose estimation

Saturday, June 5, 2021

养鱼不换水,种菜不施肥, 鱼菜共生

opencv 58 pose estimation 1

15.6 fps
#project structure
pose.py
assets
    mma.mp4
googleNet
    graph_opt.pb

#pose.py
import os
import time
import cv2 as cv
import numpy as np
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--input', default="assets/mma.mp4", help='Path to image or video. Skip to capture frames from camera')
parser.add_argument('--thr', default=0.2, type=float, help='Threshold value for pose parts heat map')
parser.add_argument('--width', default=368, type=int, help='Resize input to specific width.')
parser.add_argument('--height', default=368, type=int, help='Resize input to specific height.')

args = parser.parse_args()

BODY_PARTS = {"Nose": 0, "Neck": 1, "RShoulder": 2, "RElbow": 3, "RWrist": 4,
              "LShoulder": 5, "LElbow": 6, "LWrist": 7, "RHip": 8, "RKnee": 9,
              "RAnkle": 10, "LHip": 11, "LKnee": 12, "LAnkle": 13, "REye": 14,
              "LEye": 15, "REar": 16, "LEar": 17, "Background": 18}

POSE_PAIRS = [["Neck", "RShoulder"], ["Neck", "LShoulder"], ["RShoulder", "RElbow"],
              ["RElbow", "RWrist"], ["LShoulder", "LElbow"], ["LElbow", "LWrist"],
              ["Neck", "RHip"], ["RHip", "RKnee"], ["RKnee", "RAnkle"], ["Neck", "LHip"],
              ["LHip", "LKnee"], ["LKnee", "LAnkle"], ["Neck", "Nose"], ["Nose", "REye"],
              ["REye", "REar"], ["Nose", "LEye"], ["LEye", "LEar"]]

inWidth = args.width
inHeight = args.height

net = cv.dnn.readNetFromTensorflow("googleNet/graph_opt.pb")

cap = cv.VideoCapture(args.input if args.input else 0)

# Define the codec and create VideoWriter object
fps = 25.175
frame_width = int(cap.get(cv.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv.CAP_PROP_FRAME_HEIGHT))
size = (int(frame_width), int(frame_height))
fourcc = cv.VideoWriter_fourcc('m', 'p', '4', 'v')
path = 'C:/Users/zchen/PycharmProjects/opencv/googleNet/record'
writer = cv.VideoWriter()
success = writer.open(os.path.join(path, "pose estimation.mov"), fourcc, fps, size, True)

frame_count = 0
t1 = time.time()

while cv.waitKey(1) < 0:
    hasFrame, frame = cap.read()
    if not hasFrame:
        cv.waitKey()
        break

    frameWidth = frame.shape[1]
    frameHeight = frame.shape[0]

    net.setInput(cv.dnn.blobFromImage(frame, 1.0, (inWidth, inHeight), (127.5, 127.5, 127.5), swapRB=True, crop=False))
    out = net.forward()
    out = out[:, :19, :, :]  # MobileNet output [1, 57, -1, -1], we only need the first 19 elements

    assert (len(BODY_PARTS) == out.shape[1])

    points = []
    for i in range(len(BODY_PARTS)):
        # Slice heatmap of corresponging body's part.
        heatMap = out[0, i, :, :]

        # Originally, we try to find all the local maximums. To simplify a sample
        # we just find a global one. However only a single pose at the same time
        # could be detected this way.
        _, conf, _, point = cv.minMaxLoc(heatMap)
        x = (frameWidth * point[0]) / out.shape[3]
        y = (frameHeight * point[1]) / out.shape[2]
        # Add a point if it's confidence is higher than threshold.
        points.append((int(x), int(y)) if conf > args.thr else None)

    for pair in POSE_PAIRS:
        partFrom = pair[0]
        partTo = pair[1]
        assert (partFrom in BODY_PARTS)
        assert (partTo in BODY_PARTS)

        idFrom = BODY_PARTS[partFrom]
        idTo = BODY_PARTS[partTo]

        if points[idFrom] and points[idTo]:
            cv.line(frame, points[idFrom], points[idTo], (0, 255, 0), 3)
            cv.ellipse(frame, points[idFrom], (3, 3), 0, 0, 360, (0, 0, 255), cv.FILLED)
            cv.ellipse(frame, points[idTo], (3, 3), 0, 0, 360, (0, 0, 255), cv.FILLED)

    t, _ = net.getPerfProfile()
    freq = cv.getTickFrequency() / 1000
    cv.putText(frame, '%.2fms' % (t / freq), (10, 20), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0))

    cv.imshow("frame", frame)
    writer.write(frame)
    frame_count += 1

t2 = time.time()
# calculate FPS
fps = str(float(frame_count / float(t2 - t1))) + ' FPS'
print("Frames processed: {}".format(frame_count))
print("Elapsed time: {:.2f}".format(float(t2 - t1)))
print("FPS: {}".format(fps))

cap.release()
cv2.destroyAllWindows()

reference:

graph_opt.pb model