app.py

import streamlit as st
import numpy as np
import cv2
import tensorflow as tf 
from PIL import Image
from keras.models import load_model
from sklearn.preprocessing import LabelEncoder
import pickle
from  keras_preprocessing.sequence import pad_sequences
from  keras.preprocessing.text import Tokenizer
from sklearn.preprocessing import LabelEncoder
from PIL import Image
# from google.colab.patches import cv2_imshow

def label_smoothing(y_true,y_pred):
    
     return tf.keras.losses.binary_crossentropy(y_true,y_pred,label_smoothing=0.1)
def sparse_cross_entropy(y_true, y_pred):
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_true,
                                                          logits=y_pred)
    loss_mean = tf.reduce_mean(loss)
    return loss_mean
model1 = load_model('densenet.h5',custom_objects={'label_smoothing': label_smoothing})
image_model_transfer=load_model("image_model_transfer.h5")
decoder_model=load_model("Final_ISRO_DenseNet201_Epoch50.h5",custom_objects={'sparse_cross_entropy': sparse_cross_entropy})

class TokenizerWrap(Tokenizer):
    """Wrap the Tokenizer-class from Keras with more functionality."""
    
    def _init_(self, texts, num_words=None):
        """
        :param texts: List of strings with the data-set.
        :param num_words: Max number of words to use.
        """

        Tokenizer._init_(self, num_words=num_words)

        # Create the vocabulary from the texts.
        self.fit_on_texts(texts)

        # Create inverse lookup from integer-tokens to words.
        # word_index is a dictionary. its values are tokens and the keys are words
        # opposite to index_to_word
        self.index_to_word = dict(zip(self.word_index.values(),
                                      self.word_index.keys()))

    def token_to_word(self, token):
        """Lookup a single word from an integer-token."""
        word = " " if token == 0 else self.index_to_word[token]
        return word 

    def tokens_to_string(self, tokens):
        """Convert a list of integer-tokens to a string."""
        # Create a list of the individual words.
        words = [self.index_to_word[token]
                 for token in tokens
                 if token != 0]
        
        # Concatenate the words to a single string
        # with space between all the words.
        text = " ".join(words)

        return text
    
    def captions_to_tokens(self, captions_listlist):
        """
        Convert a list-of-list with text-captions to
        a list-of-list of integer-tokens.
        """
        
        # Note that text_to_sequences() takes a list of texts.
        tokens = [self.texts_to_sequences(captions_list)
                  for captions_list in captions_listlist]
        
        return tokens
with open('Train_Label.pickle', 'rb') as efile:
    labels=pickle.load(efile)
with open('tokenizer.pkl', 'rb') as efile:
    tokenizer=pickle.load(efile)

le=LabelEncoder()
labels=le.fit_transform(labels)

def framing(video):#defining a small function named"framing" with a parameter "i" that's supposed to be provided for reading the video
    fr = []#creating an empty list named fr
    fr_pre=[]#creating an empty list named fr_pre
    cap = cv2.VideoCapture(video)#reading the video file
    while (cap.isOpened()):#This command builds a loop to check if the data is still being read from the video
        ret,frame = cap.read()#reading the data tunnel,gives two output where one tells about presence of frames(here it's ret) & the other speaks frame data(here it's frame)
        if ret == True:#checking for presence of frames
            # cv2_imshow(frame)#displaying the frames
            grayed = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)#Converting the frames to Grayscale from BGR
            canned = cv2.Canny(grayed,320,320)#For extrating edges we use Canny Edge detection method
            fr.append(frame)#Appending the read frame
            fr_pre.append(canned)#Appending the edge extracted frames
            # cv2_imshow(grayed)#Displaying the original frames
            # cv2_imshow(canned)#Displaying the edge detected frames
            k = cv2.waitKey(10) & 0XFF#this is an arrangement for displaying the video where the secs for which each frame needs to be displayed in given in the paranthesis
            if k == ord('q'):#pressing 'q' key will close the video
                break
        else:
            break
    cap.release()#Here we release the resoures
    cv2.destroyAllWindows()#Here we delete all the windows that were created during the program       
    return fr_pre,fr 

def difference_of_frames(frames):
    diff = []#creatin a list variable
    for i in range(0,len(frames)-1):#defining the range
        diff.append(cv2.absdiff(frames[i],frames[i+1]))#appending the diff between frames to the list variable so we're supposed to get only the difference between frames
    return diff

def cal_threshold(diff):
  mn = np.mean(diff)#This gives mean
  st_d = np.std(diff)#This gives standard deviation
  a = 4#Setting a random value we can modify it to any value 
  ts = mn + (a * st_d)#defining the standard threshold value for the project/global threshold value
  return ts

def imp_frames(diff, ts, ogframes):
  a_fr = []#Creating an empty list
  for i in range(len(diff)):#Defining the for loop to be looped over all the frames obtained after finding the frames resulted from subtracting
      mn = np.mean(diff[i])#Calculating the mean for each frame
      st_d = np.std(diff[i])#Calculating the standard deviation for each frame
      fr_ts = mn + (4*st_d)#Finding the threshold values for each frame/image
      a_fr.append([i,fr_ts])#Appending the frame number & the threshold values
  imp_fr = []#Creating an empty list
  for i,ac_tr in(a_fr):#Defining the loop on the list obtained from above code
      if ac_tr >= ts:#Comapring the threshold values to the standard threshold/global threshold values
          imp_fr.append([i,ac_tr])#Appending the list with the imp frames based on their index & the values
  key_fr = []#Creating an empty list
  for i,_ in imp_fr:#Defining the loop over the list obtained from above code
      key_fr.append(ogframes[i])#This extracts the frames based on the index of frames 
  return key_fr

def final_image(video):
  frames,ogframes = framing(video)#calling function framing & then extracting the images 
  diff=difference_of_frames(frames)  
  ts=cal_threshold(diff) 
  key_fr=imp_frames(diff, ts, ogframes)  
  frame_no=key_fr[int(len(key_fr)/2)] #this is a frame
  cv2.imwrite("Testing1.jpg",frame_no)
  return "Testing1.jpg"
  cv2.destroyAllWindows()

def image_test(image_path):
  image=Image.open(image_path)
  image = image.resize((224,224)) 
  image = np.array(image) 
  image= np.expand_dims(image, axis=0)
  return image

def largest_indices(ary, n):
    flat = ary.flatten()
    indices = np.argpartition(flat, -n)[-n:]
    indices = indices[np.argsort(-flat[indices])]
    return indices

mark_start = 'ssss'
mark_end = ' eeee'

token_start = tokenizer.word_index[mark_start.strip()]
token_end = tokenizer.word_index[mark_end.strip()]

def load_image(path, size=None):
  """
  Load the image from the given file-path and resize it
  to the given size if not None.
  """

  # Load the image using PIL.
  img = Image.open(path)

  # Resize image if desired.
  if not size is None:
      img = img.resize(size=size, resample=Image.LANCZOS)
      
  img = np.array(img)
  img = img / 255.0

  # Convert 2-dim gray-scale array to 3-dim RGB array.
  if (len(img.shape) == 2):
      img = np.repeat(img[:, :, np.newaxis], 3, axis=2)
  return img

def greedy_search(image_path, max_tokens=30):
    """
    Generate a caption for the image in the given path.
    The caption is limited to the given number of tokens (words).
    """
    # ---------------------------ENCODE IMAGE--------------------------------
    # Load and resize the image.
    image = load_image(image_path, size=(224,224))
    
    # Expand the 3-dim numpy array to 4-dim
    # because the image-model expects a whole batch as input,
    # so we give it a batch with just one image.
    image_batch = np.expand_dims(image, axis=0)

    # Process the image with the pre-trained image-model
    # to get the transfer-values.
    transfer_values = image_model_transfer.predict(image_batch)
    
    # -------------------------------------------------------------------
    
    
    # Pre-allocate the 2-dim array used as input to the decoder.
    # This holds just a single sequence of integer-tokens,
    # but the decoder-model expects a batch of sequences.
    shape = (1, max_tokens)
    decoder_input_data = np.zeros(shape=shape, dtype=int)

    # The first input-token is the special start-token for 'ssss '.
    token_int = token_start #1

    # Initialize an empty output-text.
    output_text = ''

    # Initialize the number of tokens we have processed.
    count_tokens = 0

    # While we haven't sampled the special end-token for ' eeee'
    # and we haven't processed the max number of tokens.
    while token_int != token_end and count_tokens < max_tokens:
        # Update the input-sequence to the decoder
        # with the last token that was sampled.
        # In the first iteration this will set the
        # first element to the start-token.
        decoder_input_data[0, count_tokens] = token_int

        # Wrap the input-data in a dict for clarity and safety,
        # so we are sure we input the data in the right order.
        x_data = \
        {
            'transfer_values_input': transfer_values,
            'decoder_input': decoder_input_data
        }

        # Note that we input the entire sequence of tokens
        # to the decoder. This wastes a lot of computation
        # because we are only interested in the last input
        # and output. We could modify the code to return
        # the GRU-states when calling predict() and then
        # feeding these GRU-states as well the next time
        # we call predict(), but it would make the code
        # much more complicated.
        
        # Input this data to the decoder and get the predicted output.
        decoder_output = decoder_model.predict(x_data)
#         print(decoder_output.shape) (1,30,15000) for every iteration
        
        # Get the last predicted token as a one-hot encoded array.
        # Note that this is not limited by softmax, but we just
        # need the index of the largest element so it doesn't matter.
        token_onehot = decoder_output[0, count_tokens, :]
#         print(token_onehot.shape) (15000, ) for every iteration
        # Convert to an integer-token.
        token_int = np.argmax(token_onehot)
#         print(token_int) #the token of a word with the highest score
        
        # Lookup the word corresponding to this integer-token.
        sampled_word = tokenizer.token_to_word(token_int)
#         print(sampled_word)

        # Append the word to the output-text.
        output_text += " " + sampled_word

        # Increment the token-counter.
        count_tokens += 1

    # This is the sequence of tokens output by the decoder.
    output_tokens = decoder_input_data[0]
#     print(output_tokens)
    # Plot the image.
    # plt.imshow(image)
    # plt.show()

    predicted_caption=output_text.split()
    del (predicted_caption[-1])
    output_text = " "
    output_text = output_text.join(predicted_caption)

    # Print the predicted caption.
    # print("Predicted caption:")
    # print(output_text)
    # print()
    return predicted_caption

def beam_search(beam_index, image_path, max_tokens=30):
    image = load_image(image_path, size=(224,224))
    
    # Expand the 3-dim numpy array to 4-dim
    # because the image-model expects a whole batch as input,
    # so we give it a batch with just one image.
    image_batch = np.expand_dims(image, axis=0)

    # Process the image with the pre-trained image-model
    # to get the transfer-values.
    transfer_values = image_model_transfer.predict(image_batch)
    
    token_int = [token_start]
    start_word = [[token_int, 0.0]]
    count_tokens = 0 
    while len(start_word[0][0])<max_tokens:
        temp = []
        
        for s in start_word:
            par_caps = pad_sequences([s[0]], maxlen=max_tokens, padding='post')
            preds = decoder_model.predict([transfer_values,par_caps], verbose=0)
            token_onehot = preds[0, count_tokens, :]
#             print(token_onehot.shape)
            word_preds = np.argsort(token_onehot)[-beam_index:]
#             print(word_preds.shape)
            
            for w in word_preds:
                next_cap, prob = s[0][:], s[1]
                next_cap.append(w)
                prob += token_onehot[w]
                temp.append([next_cap, prob])
                  
        start_word = temp
        count_tokens+=1
        # Sorting according to the probabilities
        start_word = sorted(start_word, reverse=False, key=lambda l: l[1])
        # Getting the top words
        start_word = start_word[-beam_index:]
                  
    start_word = start_word[-1][0]
    intermediate_caption = [tokenizer.token_to_word(i) for i in start_word]
    final_caption = []
    
    for i in intermediate_caption:
        if i != 'eeee':
            final_caption.append(i)
        else:
            break

    # final_caption = ' '.join(final_caption[1:])
    return final_caption[1:]

def generate_caption_any(image_path):
  predicted_caption1=' '.join((greedy_search(image_path=image_path)))
  predicted_caption2=' '.join(beam_search(beam_index=3,image_path=image_path))
  predicted_caption3=' '.join(beam_search(beam_index=5,image_path=image_path))
  return predicted_caption2
  # show_image_using_path(image_path)



def main():
    st.title("ISRO Video Classification & Captioning")
    st.write('In this project, we introduce a technique for video classification and captioning, harnessing a keyframe extraction method to streamline the process. Utilizing Densenet 201, our model is designed to classify videos by focusing on the most crucial frame, optimizing efficiency and performance. Users can experience our innovative approach by employing any of the provided three videos which have provided as an example')

    video_options = {
            "Video 1": "Video001-Scene-001.mp4",
            "Video 2": "Video015-Scene-074.mp4",
            "Video 3": "Video005-Scene-043.mp4",
            "Video 4": "Video002-Scene-023.mp4",
        }

    selected_video = st.selectbox("Select a video to submit", list(video_options.keys()))
    video_path = video_options[selected_video]

    if st.button("Submit"):
        st.video(video_path)
        path=final_image(video_path)
        image=image_test(path)
        output_class=model1.predict(image)
        caption=generate_caption_any(path)
        indices=largest_indices(output_class, 3)
        st.title('The predicted category is:')
        st.write(le.inverse_transform(indices)[0])
        st.title('Caption:')
        caption = caption.capitalize()
        st.write(caption)
        # st.video(uploaded_file)

if __name__ == "__main__":
    main()