Harris Corner Detection

Chessboard corner detection was quite rigid, so switched to Harris.

Harris Corner Detection

#Packages
import argparse
import imutils
import datetime
import time
import cv2 as cv
import numpy as np

from imutils.video import VideoStream
from collections import deque

#Defining arguments to locate and play files
ap = argparse.ArgumentParser()
ap.add_argument("-v","--video",help="Path to the video file")
ap.add_argument("-b","--buffer",type=int,default=32,help="Max buffer size")
args = vars(ap.parse_args())

#Defining variables for the locations shown on frame
counter = 0
pts = deque(maxlen=args["buffer"])

#Reading files or use webcam to capture
if not args.get("video",False):
    v = imutils.video.VideoStream(src=0).start()
    frame_width = int(v.stream.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.stream.get(cv.CAP_PROP_FRAME_HEIGHT))
else:
    v = cv.VideoCapture(args["video"])
    frame_width = int(v.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.get(cv.CAP_PROP_FRAME_HEIGHT))

time.sleep(2.0)

index=0

file = open("res.txt","w")

#While loop to go through frames and track
while True:
    frame = v.read()
    if args.get("video", False):
        frame = frame[1]
    else:
        frame = frame

    if frame is None:
        break

    #Working the frames to be able to locate and track dot
    frame = imutils.resize(frame,width=600)
    gray = cv.cvtColor(frame,cv.COLOR_BGR2GRAY)
    
    dst = cv.cornerHarris(gray,2,3,0.04)
    dst = cv.dilate(dst,None)
    ret, dst = cv.threshold(dst,0.01*dst.max(),255,0)
    dst = np.uint8(dst)

    if ret:
        ret,labels, stats, centroids = cv.connectedComponentsWithStats(dst)
        #Define the criteria to stop and refine the corners
        criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 100, 0.001)
        corners = cv.cornerSubPix(gray,np.float32(centroids),(5,5),(-1,-1),criteria)     
        #Drawing the corners
        res = np.hstack((centroids,corners))
        res = np.int0(res)
        frame[res[:,1],res[:,0]]=[0,0,255]
        frame[res[:,3],res[:,2]] = [0,255,0]

        #Setting counter and difference in locations between frames
        for j in np.arange(1,len(pts)):
            if pts[j-1] is None or pts[j] is None:
                continue
            
            thickness = int(np.sqrt(args["buffer"]/float(j+1))*2.5)
            cv.line(frame,pts[j-1],pts[j],(0,255,0),thickness)

        #Putting text on frame to show time, and movement from last frame
        cv.putText(frame,datetime.datetime.now().strftime("%A %d %B %Y %H:%M:%S%p"),(10,30),cv.FONT_HERSHEY_TRIPLEX,0.35,(0,255,0),1)

        #Saving frames to folder
        name = './VideoSave2/frame' + str(index) + '.jpg'
        cv.imwrite(name, frame)
        index+=1

        #Show frames and give option to exit
        cv.imshow("Frame",frame)

        for k in range(len(corners)):
            res = print(corners[k][0],",",corners[k][1],file=file)

        #Saving files to folder
        for m in range(len(corners)):
            results = './Results/res' + str(m) + '.txt'
            if index==1:
                file2 = open(results,"w")
            file2 = open(results,"a")
            print(corners[m][0],",",corners[m][1],file=file2)
            file2.close()

        key = cv.waitKey(1) & 0xFF 
        counter += 1

        if key == ord("d"):
            break
    else:
        key = cv.waitKey(1) & 0xFF 
        counter += 1

        if key == ord("d"):
            break

        print("Corners not found.")
    
if not args.get("video",False):
    v.stop()

else:
    v.release()

file.close()

cv.destroyAllWindows()

Video used in this code is taken with an iPhone camera and sun lit conditions. As seen below.

chess.mp4

Using Harris because it more precisely detects corners, and it worked pretty well here. Then used the output from the corner coordinates to plot their movement. The above code is inspired by this code: https://docs.opencv.org/4.x/dc/d0d/tutorial_py_features_harris.html.

The above code writes all corner point to a file called res.txt, but does not differentiate between different points. The code for file2 writing uses this to instead write one point's location at different frame to the same file as corresponding with the frame.

The code for plotting is found below.

Plotting tracked points

import numpy as np
import matplotlib.pyplot as plt

file = []
file = (open("res.txt", "r")) #Endre filnavn så det stemmer med data fil
data = file.read().splitlines()

x,y,xn,yn = [],[],[],[]

for i in data:
    broken = i.split(",")
    xn = broken[0]
    x.append(float(xn))
    yn = broken[1]
    y.append(float(yn))

for i in range(len(x)):
    plt.plot(x[i],y[i],"o")

plt.grid()
plt.show()

file = []
file = (open("./Results/res2.txt", "r")) #Endre filnavn så det stemmer med data fil
data = file.read().splitlines()

x,y,xn,yn = [],[],[],[]

for i in data:
    broken = i.split(",")
    xn = broken[0]
    x.append(float(xn))
    yn = broken[1]
    y.append(float(yn))

for i in range(len(x)):
    plt.plot(x[i],y[i],"o")

plt.grid()
plt.show()

The result from

The points seem to jump a bit, so that some points are tracked consistently while others not at all. The first point is tracked very well, and moves approx +-5 pixels in any direction through the frames. Already on the third point tracking, we see jumping between tracking multiple different points.

Tried making the code more robust by adding limits, but it still jumps up and down in x- or y-direction. This could maybe because using a camera that moves changes the frame and isnt as rigid as it should be. Am making a setup for locating with camera standing still, and using controlled motion with the grid.

Code snippet from increasing robustness follows below.

More Robust Corner Detection

#Packages
import argparse
import imutils
import datetime
import time
import cv2 as cv
import numpy as np

from imutils.video import VideoStream
from collections import deque

#Defining arguments to locate and play files
ap = argparse.ArgumentParser()
ap.add_argument("-v","--video",help="Path to the video file")
ap.add_argument("-b","--buffer",type=int,default=32,help="Max buffer size")
args = vars(ap.parse_args())

#Defining variables for the locations shown on frame
counter = 0
pts = deque(maxlen=args["buffer"])

#Reading files or use webcam to capture
if not args.get("video",False):
    v = imutils.video.VideoStream(src=0).start()
    frame_width = int(v.stream.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.stream.get(cv.CAP_PROP_FRAME_HEIGHT))
else:
    v = cv.VideoCapture(args["video"])
    frame_width = int(v.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.get(cv.CAP_PROP_FRAME_HEIGHT))

time.sleep(2.0)

index=0

file = open("res.txt","w")

#While loop to go through frames and track
while True:
    frame = v.read()
    if args.get("video", False):
        frame = frame[1]
    else:
        frame = frame

    if frame is None:
        break

    #Working the frames to be able to locate and track dot
    frame = imutils.resize(frame,width=600)
    gray = cv.cvtColor(frame,cv.COLOR_BGR2GRAY)
    
    dst = cv.cornerHarris(gray,2,3,0.04)
    dst = cv.dilate(dst,None)
    ret, dst = cv.threshold(dst,0.01*dst.max(),255,0)
    dst = np.uint8(dst)

    if ret:
        ret,labels, stats, centroids = cv.connectedComponentsWithStats(dst)
        #Define the criteria to stop and refine the corners
        criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 100, 0.001)
        corners = cv.cornerSubPix(gray,np.float32(centroids),(5,5),(-1,-1),criteria)     
        #Drawing the corners
        res = np.hstack((centroids,corners))
        res = np.int0(res)
        frame[res[:,1],res[:,0]]=[0,0,255]
        frame[res[:,3],res[:,2]] = [0,255,0]

        #Setting counter and difference in locations between frames
        for j in np.arange(1,len(pts)):
            if pts[j-1] is None or pts[j] is None:
                continue
            
            thickness = int(np.sqrt(args["buffer"]/float(j+1))*2.5)
            cv.line(frame,pts[j-1],pts[j],(0,255,0),thickness)

        #Putting text on frame to show time, and movement from last frame
        cv.putText(frame,datetime.datetime.now().strftime("%A %d %B %Y %H:%M:%S%p"),(10,30),cv.FONT_HERSHEY_TRIPLEX,0.35,(0,255,0),1)

        #Saving frames to folder
        name = './VideoSave2/frame' + str(index) + '.jpg'
        cv.imwrite(name, frame)
        index+=1

        #Show frames and give option to exit
        cv.imshow("Frame",frame)

        for k in range(len(corners)):
            res = print(corners[k][0],",",corners[k][1],file=file)

        #Saving files to folder
        for m in range(len(corners)):
            results = './Results/restest' + str(m) + '.txt'
            if index == 1:
                filemode = "w"
            else:
                filemode = "a"

            # Write current corner's coordinates to the file
            with open(results, filemode) as file2:
                print(corners[m][0], ",", corners[m][1], file=file2)

            if m > 0:
                # Open the file again to read the last line
                file2 = open(results, "r")
                lines = file2.readlines()
                file2.close()

                if len(lines) > 1:
                    prev_line = lines[-2].strip()
                    prev_value1, prev_value2 = map(float, prev_line.split(','))

                    # Define the limit
                    limit = 10 # Adjust this as needed

                    # Check if the current corner's coordinates are within the limit of the previous coordinates
                    if abs(corners[m][0] - prev_value1) <= limit and abs(corners[m][1] - prev_value2) <= limit:
                        print("Both current values are within the limit.")
                        # Open the file to append the current corner's coordinates
                        file2 = open(results, "a")
                        print(corners[m][0], ",", corners[m][1], file=file2)
                        file2.close()
                    else:
                        print("Values are not within limits")
                else:
                    # If it's the first corner, just print a message
                    print("First corner detected. No comparison made.")
            # Close the file
            file2.close()

        key = cv.waitKey(1) & 0xFF 
        counter += 1

        if key == ord("d"):
            break
    else:
        key = cv.waitKey(1) & 0xFF 
        counter += 1

        if key == ord("d"):
            break

        print("Corners not found.")
    
if not args.get("video",False):
    v.stop()

else:
    v.release()

file.close()

cv.destroyAllWindows()

Some theory

Harris corner detection uses binary grayscale values taken from image, where large values are places with white or black, an gray are close to, or 0. A corner i found when the elliptical formed around these binary plotted points in x- and y-direction has a similarily large \lambda_1 to \lambda_2. Where \lambda are axis functions of E_{min} (1) and E_{max} (2) being largest and smallest moment of inertia. An edge comes when one of the \lambda is much larger, and a flat surface have no large \lambda. See more easy descriptions in videos such as this one. A point in a corner is where the function R is the biggest. This is also in the video.

Working code (week 7)

The code is modified to write to file correctly and track each of the points.

Tracking with Harris robust

#Packages
import argparse
import imutils
import datetime
import time
import cv2 as cv
import numpy as np

from imutils.video import VideoStream
from collections import deque

#Defining arguments to locate and play files
ap = argparse.ArgumentParser()
ap.add_argument("-v","--video",help="Path to the video file")
ap.add_argument("-b","--buffer",type=int,default=32,help="Max buffer size")
args = vars(ap.parse_args())

#Defining variables for the locations shown on frame
counter = 0
pts = deque(maxlen=args["buffer"])

#Reading files or use webcam to capture
if not args.get("video",False):
    v = imutils.video.VideoStream(src=0).start()
    frame_width = int(v.stream.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.stream.get(cv.CAP_PROP_FRAME_HEIGHT))
else:
    v = cv.VideoCapture(args["video"])
    frame_width = int(v.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.get(cv.CAP_PROP_FRAME_HEIGHT))

time.sleep(2.0)

index=0

file = open("res.txt","w")

#While loop to go through frames and track
while True:
    frame = v.read()
    if args.get("video", False):
        frame = frame[1]
    else:
        frame = frame

    if frame is None:
        break

    #Working the frames to be able to locate and track dot
    frame = imutils.resize(frame,width=600)
    gray = cv.cvtColor(frame,cv.COLOR_BGR2GRAY)
    
    dst = cv.cornerHarris(gray,2,3,0.04)
    dst = cv.dilate(dst,None)
    ret, dst = cv.threshold(dst,0.01*dst.max(),255,0)
    dst = np.uint8(dst)

    if ret:
        ret,labels, stats, centroids = cv.connectedComponentsWithStats(dst)
        #Define the criteria to stop and refine the corners
        criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 100, 0.001)
        corners = cv.cornerSubPix(gray,np.float32(centroids),(5,5),(-1,-1),criteria)     
        #Drawing the corners
        res = np.hstack((centroids,corners))
        res = np.int0(res)
        frame[res[:,1],res[:,0]]=[0,0,255]
        frame[res[:,3],res[:,2]] = [0,255,0]

        #Setting counter and difference in locations between frames
        for j in np.arange(1,len(pts)):
            if pts[j-1] is None or pts[j] is None:
                continue
            
            thickness = int(np.sqrt(args["buffer"]/float(j+1))*2.5)
            cv.line(frame,pts[j-1],pts[j],(0,255,0),thickness)

        #Putting text on frame to show time, and movement from last frame
        cv.putText(frame,datetime.datetime.now().strftime("%A %d %B %Y %H:%M:%S%p"),(10,30),cv.FONT_HERSHEY_TRIPLEX,0.35,(0,255,0),1)

        #Saving frames to folder
        name = './VideoSave2/frame' + str(index) + '.jpg'
        cv.imwrite(name, frame)
        index+=1

        #Show frames and give option to exit
        cv.imshow("Frame",frame)

        for k in range(len(corners)):
            res = print(corners[k][0],",",corners[k][1],file=file)

        #Saving files to folder
        for m in range(len(corners)):
            results = './Results/restest' + str(m) + '.txt'
            if index == 1:
                file2 = open(results, "w")
                print(corners[m][0], ",", corners[m][1], file=file2)    
            else:
                file2 = open(results, "a")

                # Open the file to read the last line
                file2 = open(results, "r")
                lines = file2.readlines()
                file2.close()

                if len(lines) > 0:
                    prev_line = lines[-1].strip()
                    prev_value1,prev_value2 = prev_line.split(",")
                    prev_value1 = float(prev_value1)
                    prev_value2 = float(prev_value2)

                    # Define the limit
                    limit = float(10) # Adjust this as needed

                    # Check if the current corner's coordinates are within the limit of the previous coordinates
                    if (abs(corners[m][0] - prev_value1) <= limit) and (abs(corners[m][1] - prev_value2) <= limit):
                        print("Both current values are within the limit.")
                        # Open the file to append the current corner's coordinates
                        file2 = open(results, "a")
                        print(corners[m][0], ",", corners[m][1], file=file2)
                        file2.close()
                    else:
                        print("Values are not within limits")
                else:
                    print("First corner detected. No comparison made.")

        # Close the file after the loop finishes
        if file2:
            file2.close()

        key = cv.waitKey(1) & 0xFF 
        counter += 1

        if key == ord("d"):
            break
    else:
        key = cv.waitKey(1) & 0xFF 
        counter += 1

        if key == ord("d"):
            break

        print("Corners not found.")
    
if not args.get("video",False):
    v.stop()

else:
    v.release()

file.close()

cv.destroyAllWindows()

The code still adds more points to the first list than the rest (313, to about 80). So there is still need for more filtering.

As you can see, the movement is pretty similar in the different points, but the amount of points vary significantly. This should be possible to fix by adjusting the limits.

The reason why some points have barely any values, are because they arent really corners, but points that pop up on random locations. The image below shows an untuned version of the video.

In this frame, only the correct corners are marked, but sometimes points appear both within the grid, and outside it.

To manually change this thresholding, the limits and k-value in the cv.treshold function is tweaked. Then use the dst frame to locate points that appear and disappear. The point below appears and disappears multiple times during the iterations through frames.

TrackCalibrating

#Packages
import argparse
import imutils
import datetime
import time
import cv2 as cv
import numpy as np

from imutils.video import VideoStream
from collections import deque

#Defining arguments to locate and play files
ap = argparse.ArgumentParser()
ap.add_argument("-v","--video",help="Path to the video file")
ap.add_argument("-b","--buffer",type=int,default=32,help="Max buffer size")
args = vars(ap.parse_args())

#Defining variables for the locations shown on frame
counter = 0
pts = deque(maxlen=args["buffer"])

#Reading files or use webcam to capture
if not args.get("video",False):
    v = imutils.video.VideoStream(src=0).start()
    frame_width = int(v.stream.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.stream.get(cv.CAP_PROP_FRAME_HEIGHT))
else:
    v = cv.VideoCapture(args["video"])
    frame_width = int(v.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.get(cv.CAP_PROP_FRAME_HEIGHT))

time.sleep(2.0)

index=0

file = open("res.txt","w")

#While loop to go through frames and track
while True:
    frame = v.read()
    if args.get("video", False):
        frame = frame[1]
    else:
        frame = frame

    if frame is None:
        break

    #Working the frames to be able to locate and track dot
    frame = imutils.resize(frame,width=600)
    gray = cv.cvtColor(frame,cv.COLOR_BGR2GRAY)
    
    dst = cv.cornerHarris(gray,2,3,0.07)
    dst = cv.dilate(dst,None)
    ret, dst = cv.threshold(dst,0.01*dst.max(),255,0)
    dst = np.uint8(dst)
    cv.imshow("dst",dst)

    if ret:
        ret,labels, stats, centroids = cv.connectedComponentsWithStats(dst)
        #Define the criteria to stop and refine the corners
        criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 100, 0.001)
        corners = cv.cornerSubPix(gray,np.float32(centroids),(50,50),(-1,-1),criteria)     
        #Drawing the corners
        res = np.hstack((centroids,corners))
        res = np.int0(res)
        frame[res[:,1],res[:,0]]=[0,0,255]
        frame[res[:,3],res[:,2]] = [0,255,0]

        #Setting counter and difference in locations between frames
        for j in np.arange(1,len(pts)):
            if pts[j-1] is None or pts[j] is None:
                continue
            
            thickness = int(np.sqrt(args["buffer"]/float(j+1))*2.5)
            cv.line(frame,pts[j-1],pts[j],(0,255,0),thickness)

        #Putting text on frame to show time, and movement from last frame
        cv.putText(frame,datetime.datetime.now().strftime("%A %d %B %Y %H:%M:%S%p"),(10,30),cv.FONT_HERSHEY_TRIPLEX,0.35,(0,255,0),1)

        #Saving frames to folder
        name = './VideoSave2/frame' + str(index) + '.jpg'
        cv.imwrite(name, frame)
        threshimg = "./Thresh/frame" + str(index) + ".jpg"
        cv.imwrite(threshimg,dst)
        index+=1

        #Show frames and give option to exit
        cv.imshow("Frame",frame)

        for k in range(len(corners)):
            res = print(corners[k][0],",",corners[k][1],file=file)

        #Saving files to folder
        for m in range(len(corners)):
            results = './Results/restest' + str(m) + '.txt'
            if index == 1:
                file2 = open(results, "w")
                print(corners[m][0], ",", corners[m][1], file=file2)    
            else:
                file2 = open(results, "a")

                # Open the file to read the last line
                file2 = open(results, "r")
                lines = file2.readlines()
                file2.close()

                if len(lines) > 0:
                    prev_line = lines[-1].strip()
                    prev_value1,prev_value2 = prev_line.split(",")
                    prev_value1 = float(prev_value1)
                    prev_value2 = float(prev_value2)

                    # Define the limit
                    limit = float(50) # Adjust this as needed

                    # Check if the current corner's coordinates are within the limit of the previous coordinates
                    if (abs(corners[m][0] - prev_value1) <= limit) and (abs(corners[m][1] - prev_value2) <= limit):
                        print("Both current values are within the limit.")
                        # Open the file to append the current corner's coordinates
                        file2 = open(results, "a")
                        print(corners[m][0], ",", corners[m][1], file=file2)
                        file2.close()
                    else:
                        print("Values are not within limits")
                else:
                    print("First corner detected. No comparison made.")

        # Close the file after the loop finishes
        if file2:
            file2.close()

        key = cv.waitKey(1) & 0xFF 
        counter += 1

        if key == ord("d"):
            break
    else:
        key = cv.waitKey(1) & 0xFF 
        counter += 1

        if key == ord("d"):
            break

        print("Corners not found.")
    
if not args.get("video",False):
    v.stop()

else:
    v.release()

file.close()

cv.destroyAllWindows()

As seen below, there should be 132 points in the frame, and there are 312 frames. So there should be 132 files with points, and 313 lines of coordinates within these files. Currently, 133 files are found, and the lines vary from about 34 to 313, and most have about 100. There should be the double amount of lines in the files with two points close to each other.

Later switched from chess to grid, and adjusted the k-value to 0.06 (includes at least one jumper point). This was 207 frames and 96 points. But 104 files are made.

To find a fitting limit, I plotted all first values in files, and found points beside each other and calculated distance. This gave these results:

The first 64 frames (except the first frame) now have the 207 points they should have. The code is becoming increasingly robust.

Multilayer Grids

Have a new code for tracking grids on two different layers. Still working on som bugs, but the mask i really clear and good now. Below are the two masks and the original image.

Even though the code masks pretty good, the corners are not detected perfectly, with the output corner positions being plotted below:

Changed code from external to tree, in contour detection, and got:

TrackingTwoGrids

import cv2
import numpy as np
import matplotlib.pyplot as plt

# Load the image
image = cv2.imread('grids/Grids.png')

# Convert image to grayscale
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
plt.imshow(gray)

# Harris corner detection
corners = cv2.cornerHarris(gray, 2, 3, 0.04)

# Threshold corners to get strong corners
threshold = 0.01 * corners.max()

# Mark corners belonging to the first grid (e.g., red) in red
image_with_corners = image.copy()
image_with_corners[corners > threshold] = [0, 0, 255]

plt.imshow(image_with_corners)

# Mark corners belonging to the second grid (e.g., green) in green
# Assuming the second grid is green, you may need to adjust the color range
green_lower = np.array([35, 50, 50], dtype=np.uint8)
green_upper = np.array([85, 255, 255], dtype=np.uint8)
red_lower = np.array([0, 100, 100], np.uint8) 
red_upper = np.array([10, 255, 255], np.uint8) 

#Check of boundaries
blank = np.zeros_like(image)
blank[:] = green_lower
plt.imshow(blank)
plt.show()

hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
mask_green = cv2.inRange(hsv, green_lower, green_upper)
mask_green = cv2.dilate(mask_green,None,iterations=2)
mask_red = cv2.inRange(hsv,red_lower,red_upper)
mask_red = cv2.dilate(mask_red,None,iterations=2)

plt.imshow(mask_red)
plt.show()
plt.imshow(mask_green)
plt.show()

contours1, _ = cv2.findContours(mask_green, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
contours2, _ = cv2.findContours(mask_red,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)

for contour in contours1:
    x = [point[0][0] for point in contour]  # Extract x coordinates from each point in the contour
    y = [point[0][1] for point in contour]  # Extract y coordinates from each point in the contour
    plt.plot(x, y,"o", color='green',markersize=0.5)  # Plot the contour

# Plot contours from contours2
for contour in contours2:
    x = [point[0][0] for point in contour]  # Extract x coordinates from each point in the contour
    y = [point[0][1] for point in contour]  # Extract y coordinates from each point in the contour
    plt.plot(x, y,"o", color='red',markersize=0.5)  # Plot the contour

# Set plot labels and title
plt.xlabel('X-axis')
plt.ylabel('Y-axis')
plt.title('Contours Plot')

# Show the plot
plt.grid(True)
plt.gca().invert_yaxis()  # Invert y-axis to match the image coordinates
plt.show()

Video grid tracking without Harris

Tracking two grids video

#Packages
import argparse
import imutils
import datetime
import time
import cv2 as cv
import numpy as np
import matplotlib.pyplot as plt

from imutils.video import VideoStream
from collections import deque
import imutils

#Defining arguments to locate and play files
ap = argparse.ArgumentParser()
ap.add_argument("-v","--video",help="Path to the video file")
ap.add_argument("-b","--buffer",type=int,default=32,help="Max buffer size")
args = vars(ap.parse_args())

counter = 0
pts1 = deque(maxlen=args["buffer"])
pts2 = deque(maxlen=args["buffer"])


green_lower = np.array([50, 40, 40])    # Lower boundary for green
green_upper = np.array([90, 255, 255])  # Upper boundary for green
#green_lower = np.array([35, 50, 50], dtype=np.uint8)
#green_upper = np.array([85, 255, 255], dtype=np.uint8)
red_lower = np.array([0, 100, 100], np.uint8) 
red_upper = np.array([10, 255, 255], np.uint8) 


#Reading files or use webcam to capture
if not args.get("video",False):
    v = imutils.video.VideoStream(src=0).start()
    frame_width = int(v.stream.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.stream.get(cv.CAP_PROP_FRAME_HEIGHT))
else:
    v = cv.VideoCapture(args["video"])
    frame_width = int(v.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.get(cv.CAP_PROP_FRAME_HEIGHT))

time.sleep(2.0)

index=0

while True:
    frame = v.read()
    if args.get("video", False):
        frame = frame[1]
    else:
        frame = frame

    if frame is None:
        break

    #Working the frames to be able to locate and track red dot
    hsv = cv.cvtColor(frame, cv.COLOR_BGR2HSV)
    blur = cv.GaussianBlur(hsv,(5,5),6)
    mask_green = cv.inRange(blur, green_lower, green_upper)
    mask_green = cv.dilate(mask_green,None,iterations=2)
    mask_red = cv.inRange(blur,red_lower,red_upper)
    mask_red = cv.dilate(mask_red,None,iterations=2)

    cv.imshow("Red",mask_red)
    #plt.show()
    cv.imshow("Green",mask_green)
    #plt.show()

    contours1, _ = cv.findContours(mask_green.copy(), cv.RETR_TREE, cv.CHAIN_APPROX_SIMPLE)
    contours2, _ = cv.findContours(mask_red.copy(),cv.RETR_TREE,cv.CHAIN_APPROX_SIMPLE)

    

    for contour in contours1:
        x = [point[0][0] for point in contour]  # Extract x coordinates from each point in the contour
        y = [point[0][1] for point in contour]  # Extract y coordinates from each point in the contour
        #plt.plot(x, y,"o", color="green",markersize=0.5) # Plot the contour

    # Plot contours from contours2
    for contour in contours2:
        x = [point[0][0] for point in contour]  # Extract x coordinates from each point in the contour
        y = [point[0][1] for point in contour]  # Extract y coordinates from each point in the contour
        #plt.plot(x, y,"o", color="red",markersize=0.5)  # Plot the contour
    # Set plot labels and title
    #plt.xlabel('X-axis')
    #plt.ylabel('Y-axis')
    #plt.title('Contours Plot')

    # Show the plot
    #plt.grid(True)
    #plt.gca().invert_yaxis()  # Invert y-axis to match the image coordinates
    #plt.show()
    for m in range(len(contours1)):
        results = './ResultsTwoGrids/resc1_' + str(m) + '.csv'
        if index==1:
            file2 = open(results,"w")
        file2 = open(results,"a")
        print(contours1[m][0],",",contours1[m][1],file=file2)
        file2.close()

    for n in range(len(contours2)):
        results = './ResultsTwoGrids/res' + str(n) + '.csv'
        if index==1:
            file2 = open(results,"w")
        file2 = open(results,"a")
        print(contours2[n][0],",",contours2[n][1],file=file2)
        file2.close()

    cv.imshow("Frame",frame)

    key = cv.waitKey(1) & 0xFF 
    counter += 1

    if key == ord("d"):
        break

if not args.get("video",False):
    v.stop()

else:
    v.release()

cv.destroyAllWindows()

This video locates corners in video, however it mostly does so by tracing the grid not finding the exact corners. Am working on a new one that should be able to do so.

Tracking video with Harris

This locates way too many corners, so need blurring and correcting to make it more robust.

Harris video

import argparse
import imutils
import datetime
import time
import cv2 as cv
import numpy as np

from imutils.video import VideoStream
from collections import deque

#Defining arguments to locate and play files
ap = argparse.ArgumentParser()
ap.add_argument("-v","--video",help="Path to the video file")
ap.add_argument("-b","--buffer",type=int,default=32,help="Max buffer size")
args = vars(ap.parse_args())

#Defining variables for the locations shown on frame
counter = 0
pts = deque(maxlen=args["buffer"])

green_lower = np.array([70, 40, 40])    # Lower boundary for green
green_upper = np.array([110, 255, 255])  # Upper boundary for green
#green_lower = np.array([35, 50, 50], dtype=np.uint8)
#green_upper = np.array([85, 255, 255], dtype=np.uint8)
red_lower = np.array([0, 100, 100], np.uint8) 
red_upper = np.array([10, 255, 255], np.uint8) 



#Reading files or use webcam to capture
if not args.get("video",False):
    v = imutils.video.VideoStream(src=0).start()
    frame_width = int(v.stream.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.stream.get(cv.CAP_PROP_FRAME_HEIGHT))
else:
    v = cv.VideoCapture(args["video"])
    frame_width = int(v.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.get(cv.CAP_PROP_FRAME_HEIGHT))

time.sleep(2.0)


while True:
    frame = v.read()
    if args.get("video", False):
        frame = frame[1]
    else:
        frame = frame

    if frame is None:
        break

    frame = imutils.resize(frame,width=600)
    gray = cv.cvtColor(frame,cv.COLOR_BGR2GRAY)
    corners = cv.cornerHarris(gray,2,3,0.04)

    hsv = cv.cvtColor(frame, cv.COLOR_BGR2HSV)
    #blur = cv.GaussianBlur(hsv,(5,5),9)
    mask_green = cv.inRange(hsv, green_lower, green_upper)
    mask_green = cv.dilate(mask_green,None,iterations=2)
    mask_red = cv.inRange(hsv,red_lower,red_upper)
    mask_red = cv.dilate(mask_red,None,iterations=2)

    corners1 = cv.cornerHarris(mask_red,2,3,0.04)
    corners2 = cv.cornerHarris(mask_green,2,3,0.04)

    thresh1 = 0.01*corners1.max()
    thresh2 = 0.01*corners2.max()

    frame_w_corners_red = frame.copy()
    frame_w_corners_red[corners1 > thresh1] = [0, 0, 255]

    frame_w_corners_green = frame.copy()
    frame_w_corners_green[corners2 > thresh2] = [0, 255, 0]

    cv.imshow("Green",frame_w_corners_green)
    cv.imshow("Red",frame_w_corners_red)

    key = cv.waitKey(1) & 0xFF 

    if key == ord("d"):
        break

if not args.get("video",False):
    v.stop()

else:
    v.release()

cv.destroyAllWindows()

The results from corner detection is seen below

Did some alterations resulting in this code:

More robust video harris

import argparse
import imutils
import datetime
import time
import cv2 as cv
import numpy as np

from imutils.video import VideoStream
from collections import deque

#Defining arguments to locate and play files
ap = argparse.ArgumentParser()
ap.add_argument("-v","--video",help="Path to the video file")
ap.add_argument("-b","--buffer",type=int,default=32,help="Max buffer size")
args = vars(ap.parse_args())

#Defining variables for the locations shown on frame
counter = 0
pts = deque(maxlen=args["buffer"])

green_lower = np.array([70, 40, 40])    # Lower boundary for green
green_upper = np.array([110, 255, 255])  # Upper boundary for green
red_lower = np.array([0, 100, 100], np.uint8) 
red_upper = np.array([10, 255, 255], np.uint8) 



#Reading files or use webcam to capture
if not args.get("video",False):
    v = imutils.video.VideoStream(src=0).start()
    frame_width = int(v.stream.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.stream.get(cv.CAP_PROP_FRAME_HEIGHT))
else:
    v = cv.VideoCapture(args["video"])
    frame_width = int(v.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.get(cv.CAP_PROP_FRAME_HEIGHT))

time.sleep(2.0)


while True:
    frame = v.read()
    if args.get("video", False):
        frame = frame[1]
    else:
        frame = frame

    if frame is None:
        break

    frame = imutils.resize(frame,width=600)
    gray = cv.cvtColor(frame,cv.COLOR_BGR2GRAY)
    corners = cv.cornerHarris(gray,2,3,0.04)

    hsv = cv.cvtColor(frame, cv.COLOR_BGR2HSV)
    #blur = cv.GaussianBlur(hsv,(5,5),9)
    mask_green = cv.inRange(hsv, green_lower, green_upper)
    mask_green = cv.dilate(mask_green,None,iterations=2)
    mask_red = cv.inRange(hsv,red_lower,red_upper)
    mask_red = cv.dilate(mask_red,None,iterations=2)

    dst1 = cv.cornerHarris(mask_green,2,3,0.04)
    dst2 = cv.cornerHarris(mask_red,2,3,0.04)
    dst1 = cv.dilate(dst1,None)
    dst2 = cv.dilate(dst2,None)
    ret, dst1 = cv.threshold(dst1,0.01*dst1.max(),255,0)
    ret, dst2 = cv.threshold(dst2,0.01*dst2.max(),255,0)
    dst1 = np.uint8(dst1)
    dst2 = np.uint8(dst2)
 
    if ret:
        ret1,labels1, stats1, centroids1 = cv.connectedComponentsWithStats(dst1)
        ret2,labels2, stats2, centroids2 = cv.connectedComponentsWithStats(dst2)
        #Define the criteria to stop and refine the corners
        criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 100, 0.001)
        corners1 = cv.cornerSubPix(gray,np.float32(centroids1),(5,5),(-1,-1),criteria)
        corners2 = cv.cornerSubPix(gray,np.float32(centroids2),(5,5),(-1,-1),criteria)   
        #Drawing the corners
        res1 = np.hstack((centroids1,corners1))
        res2 = np.hstack((centroids2,corners2))
        res1 = np.int0(res1)
        res2 = np.int0(res2)

        frame_w_corners_red = frame.copy()
        frame_w_corners_green = frame.copy()
        frame_w_corners_green[res1[:,1],res1[:,0]]=[0,255,0]
        frame_w_corners_green[res1[:,3],res1[:,2]] = [0,255,0]
        frame_w_corners_red[res2[:,1],res2[:,0]]=[0,0,255]
        frame_w_corners_red[res2[:,3],res2[:,2]] = [0,0,255]
    

    cv.imshow("Green",frame_w_corners_green)
    cv.imshow("Red",frame_w_corners_red)

    key = cv.waitKey(1) & 0xFF 

    if key == ord("d"):
        break

if not args.get("video",False):
    v.stop()

else:
    v.release()

cv.destroyAllWindows()

Harris new better and best for now

Newest or something

import argparse
import imutils
import datetime
import time
import cv2 as cv
import numpy as np

from imutils.video import VideoStream
from collections import deque

cv.namedWindow('Tracking')
def nothing(x): 
    pass

#Defining arguments to locate and play files
ap = argparse.ArgumentParser()
ap.add_argument("-v","--video",help="Path to the video file")
ap.add_argument("-b","--buffer",type=int,default=32,help="Max buffer size")
args = vars(ap.parse_args())

#Defining upper and lower bounds to locate the dots
#Set trackbar to change value of red on frame
cv.createTrackbar("LH", "Tracking", 
                   0, 255, nothing) 
cv.createTrackbar("LS", "Tracking",  
                   0, 255, nothing) 
cv.createTrackbar("LV", "Tracking",  
                   0, 255, nothing) 
cv.createTrackbar("HH", "Tracking",  
                   0, 255, nothing) 
cv.createTrackbar("HS", "Tracking",  
                   0, 255, nothing) 
cv.createTrackbar("HV", "Tracking", 
                   0, 255, nothing) 

#Defining variables for the locations shown on frame
counter = 0
pts = deque(maxlen=args["buffer"])

green_lower = np.array([62, 57, 64])    # Lower boundary for green
green_upper = np.array([153, 255, 251])  # Upper boundary for green
#green_lower = np.array([35, 50, 50], dtype=np.uint8)
#green_upper = np.array([85, 255, 255], dtype=np.uint8)
red_lower = np.array([0, 87, 64], np.uint8) 
red_upper = np.array([23, 255, 255], np.uint8) 



#Reading files or use webcam to capture
if not args.get("video",False):
    v = imutils.video.VideoStream(src=0).start()
    frame_width = int(v.stream.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.stream.get(cv.CAP_PROP_FRAME_HEIGHT))
else:
    v = cv.VideoCapture(args["video"])
    frame_width = int(v.get(cv.CAP_PROP_FRAME_WIDTH))
    frame_height = int(v.get(cv.CAP_PROP_FRAME_HEIGHT))

time.sleep(2.0)

index=0

file = open("./ResultsTwice/Gres.txt","w")
file2 = open("./ResultsTwice/Rres.txt","w")


while True:
    frame = v.read()
    if args.get("video", False):
        frame = frame[1]
    else:
        frame = frame

    if frame is None:
        break

    frame = imutils.resize(frame,width=600)
    gray = cv.cvtColor(frame,cv.COLOR_BGR2GRAY)
    corners = cv.cornerHarris(gray,2,3,0.04)

    hsv = cv.cvtColor(frame, cv.COLOR_BGR2HSV)
    blur = cv.GaussianBlur(hsv,(5,5),9)

    l_h = cv.getTrackbarPos("LH", "Tracking") 
    l_s = cv.getTrackbarPos("LS", "Tracking") 
    l_v = cv.getTrackbarPos("LV", "Tracking") 
    h_h = cv.getTrackbarPos("HH", "Tracking") 
    h_s = cv.getTrackbarPos("HS", "Tracking") 
    h_v = cv.getTrackbarPos("HV", "Tracking") 
    l_b = np.array([l_h, l_s, l_v]) 
    u_b = np.array([h_h, h_s, h_v]) 

    mask_green = cv.inRange(hsv, green_lower, green_upper)
    mask_green = cv.dilate(mask_green,None,iterations=2)
    mask_red = cv.inRange(hsv,red_lower,red_upper)
    mask_red = cv.dilate(mask_red,None,iterations=2)

    dst1 = cv.cornerHarris(mask_green,2,3,0.16)
    dst2 = cv.cornerHarris(mask_red,2,3,0.2)
    dst1 = cv.dilate(dst1,None)
    dst2 = cv.dilate(dst2,None)
    ret, dst1 = cv.threshold(dst1,0.01*dst1.max(),255,0)
    ret, dst2 = cv.threshold(dst2,0.01*dst2.max(),255,0)
    dst1 = np.uint8(dst1)
    dst2 = np.uint8(dst2)
 
    if ret:
        ret1,labels1, stats1, centroids1 = cv.connectedComponentsWithStats(dst1)
        ret2,labels2, stats2, centroids2 = cv.connectedComponentsWithStats(dst2)
        #Define the criteria to stop and refine the corners
        criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 100, 0.001)
        corners1 = cv.cornerSubPix(gray,np.float32(centroids1),(5,5),(-1,-1),criteria)
        corners2 = cv.cornerSubPix(gray,np.float32(centroids2),(5,5),(-1,-1),criteria)   
        #Drawing the corners
        res1 = np.hstack((centroids1,corners1))
        res2 = np.hstack((centroids2,corners2))
        res1 = np.int0(res1)
        res2 = np.int0(res2)

        frame_w_corners_red = frame.copy()
        frame_w_corners_green = frame.copy()
        frame_w_corners_green[res1[:,1],res1[:,0]]=[0,255,0]
        frame_w_corners_green[res1[:,3],res1[:,2]] = [0,255,0]
        frame_w_corners_red[res2[:,1],res2[:,0]]=[0,0,255]
        frame_w_corners_red[res2[:,3],res2[:,2]] = [0,0,255]
        
        #Saving files to folder
        for k in range(len(corners1)):
            res = print(corners1[k][0],",",corners1[k][1],file=file)

        for m in range(len(corners2)):
            res = print(corners2[m][0],",",corners2[m][1],file=file2)
 
    cv.imshow("Green",frame_w_corners_green)
    cv.imshow("Red",frame_w_corners_red)
    #cv.imshow("Green",mask_green)
    #cv.imshow("Red",mask_red)
    key = cv.waitKey(1) & 0xFF 

    if key == ord("d"):
        break

if not args.get("video",False):
    v.stop()

else:
    v.release()

file.close()
file2.close()

cv.destroyAllWindows()

It still detects way too many corners, which is very not ideal, but it sketches the whole grid pretty good. Could also talk to Eirik on what to do to limit the points to only the corners. It works for digital images, but not videos of physical environment.

Video file: Both (1) (1).mp4.

Next is filtering and getting the individual points in the corners for both layers.

Space shortcuts

Page tree