Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
*.txt
*.png
*.jpg
*.pgf
*.xml
*.avi
*.mp4
Expand Down
3 changes: 2 additions & 1 deletion examples/visual_voice_activity_detection/live_demo.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
from paz.backend.camera import VideoPlayer, Camera
import paz.pipelines.detection as dt

parser = argparse.ArgumentParser(description='Visual Voice Activity Detection Live Demonstration')
parser = argparse.ArgumentParser(description='Visual Voice Activity Detection '
'Live Demonstration')
parser.add_argument('-c', '--camera_id', type=int, default=0,
help='Camera device ID')
args = parser.parse_args()
Expand Down
15 changes: 10 additions & 5 deletions examples/visual_voice_activity_detection/recorded_demo.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,16 @@
from paz.backend.camera import VideoPlayer
import paz.pipelines.detection as dt

parser = argparse.ArgumentParser(description='Visual Voice Activity Detection Recorded Demonstration')
parser.add_argument('-i', '--input_path', type=str, default="./demo_video.mp4",
help='Path to the video file to be used as input for the VVAD Pipeline.')
parser.add_argument('-o', '--output_path', type=str, default="./demo_video_labeled.avi",
help='Path to the video file to be used as output for the VVAD Pipeline.')
parser = argparse.ArgumentParser(description='Visual Voice Activity Detection '
'Recorded Demonstration')
parser.add_argument('-i', '--input_path', type=str,
default="./demo_video.mp4",
help='Path to the video file to be used as input for the '
'VVAD Pipeline.')
parser.add_argument('-o', '--output_path', type=str,
default="./demo_video_labeled.avi",
help='Path to the video file to be used as output for the '
'VVAD Pipeline.')
args = parser.parse_args()

pipeline = dt.DetectVVAD()
Expand Down
107 changes: 68 additions & 39 deletions examples/visual_voice_activity_detection/vvad_lrs3_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,41 +16,52 @@ class VvadLrs3Dataset(Generator):
# Arguments
path: String. Full path to vvadlrs3_faceImages_small.h5 file.
split: String. Valid option contain 'train', 'validation' or 'test'.
validation_split: Float. Percentage of the dataset to be used for validation (valid options between 0.0 to 1.0). Set
to 0.0 to disable.
test_split: Float. Percentage of the dataset to be used for testing (valid options between 0.0 to 1.0). Set
to 0.0 to disable.
testing: Boolean. If True, a smaller dataset of 200 samples is used for testing. This is useful for testing
evaluating: Boolean. If True, the dataset is used for evaluation. This means that the dataset is not shuffled
and the indexes will be stored.
reduction_method: String. Valid options are 'cut' or 'reduce'. If 'cut' is selected, the video is cut to the
reduction_length. If 'reduce' is selected, reduction_length many single frames of the video is removed form
the clip.
reduced_length: Float. The length of the video after the reduction_method is applied. Choose None if you
validation_split: Float. Percentage of the dataset to be used for
validation (valid options between 0.0 to 1.0).
Set to 0.0 to disable.
test_split: Float. Percentage of the dataset to be used for testing
(valid options between 0.0 to 1.0).
Set to 0.0 to disable.
testing: Boolean. If True, a smaller dataset of 200 samples is used
for testing. This is useful for testing
evaluating: Boolean. If True, the dataset is used for evaluation.
This means that the dataset is not shuffled and the indexes will
be stored.
reduction_method: String. Valid options are 'cut' or 'reduce'.
If 'cut' is selected, the video is cut to the reduction_length.
If 'reduce' is selected, reduction_length many single frames of
the video is removed form the clip.
reduced_length: Float. The length of the video after
the reduction_method is applied. Choose None if you
want to keep the original size. None is the default

# References
-[VVAD-LRS3](https://www.kaggle.com/datasets/adrianlubitz/vvadlrs3)
"""
def __init__(
self, path='.keras/paz/datasets', split='train', validation_split=0.2, test_split=0.1, testing=False,
evaluating=False, reduction_method: Reduction_Method = 'cut', reduced_length=None):
self, path='.keras/paz/datasets', split='train',
validation_split=0.2, test_split=0.1, testing=False,
evaluating=False, reduction_method: Reduction_Method = 'cut',
reduced_length=None):
if split != 'train' and split != 'validation' and split != 'test':
raise ValueError('Invalid split name')
if validation_split < 0.0 or validation_split > 1.0:
raise ValueError('Invalid validation split')
if test_split < 0.0 or test_split > 1.0:
raise ValueError('Invalid test split')
if validation_split + test_split > 1.0:
raise ValueError('The sum of val_split and test_split must be less than 1.0')
raise ValueError('The sum of val_split and test_split must be '
'less than 1.0')
options = get_args(Reduction_Method)
assert reduction_method in options, f"'{reduction_method}' is not in {options}"
assert reduction_method in options, \
f"'{reduction_method}' is not in {options}"

path = os.path.join(path, 'vvadlrs3_faceImages_small.h5')

class_names = get_class_names('VVAD_LRS3')

super(VvadLrs3Dataset, self).__init__(path, split, class_names, 'VVAD_LRS3')
super(VvadLrs3Dataset, self).__init__(path, split, class_names,
'VVAD_LRS3')
self.validation_split = validation_split
self.test_split = test_split
self.use_test_data = testing
Expand All @@ -60,10 +71,12 @@ def __init__(
self.reduced_length = reduced_length

# Get total dataset size and length of the video clips
self.total_dataset_size, self.video_length = self.get_data_and_video_size()
self.total_dataset_size, self.video_length \
= self.get_data_and_video_size()

# Split dataset indexes
self.indexes_train, self.indexes_validation, self.indexes_test = self.split_data_indices()
self.indexes_train, self.indexes_validation, self.indexes_test \
= self.split_data_indices()

# Reduction init
self.dropout_ids = []
Expand All @@ -75,7 +88,7 @@ def __call__(self):
indexes = []
if self.split == 'train':
indexes = self.indexes_train
random.shuffle(indexes) # Use always the same seed so every model gets the same shuffle
random.shuffle(indexes)
elif self.split == 'validation':
indexes = self.indexes_validation
elif self.split == 'test':
Expand All @@ -95,7 +108,8 @@ def __call__(self):
data.close()

def get_data_and_video_size(self):
"""Gets the total size of the dataset and the length of the video clips."""
"""Gets the total size of the dataset and the length of the video clips.
"""
data = h5py.File(self.path, mode='r')

total_size = 0
Expand All @@ -111,7 +125,8 @@ def get_data_and_video_size(self):
def split_data_indices(self):
"""Splits the data indices into train, validation and test."""
indexes_positive = list(range(self.total_dataset_size // 2))
indexes_negative = list(range(self.total_dataset_size // 2, self.total_dataset_size))
indexes_negative = list(range(self.total_dataset_size // 2,
self.total_dataset_size))

random.Random(445363).shuffle(indexes_positive)
random.Random(848641).shuffle(indexes_negative)
Expand All @@ -121,10 +136,12 @@ def split_data_indices(self):

if self.validation_split > 0.0:
indexes_validation, indexes_positive, indexes_negative = \
self.split_by_ratio(self.validation_split, indexes_positive, indexes_negative)
self.split_by_ratio(self.validation_split, indexes_positive,
indexes_negative)
if self.test_split > 0.0:
indexes_test, indexes_positive, indexes_negative = \
self.split_by_ratio(self.test_split, indexes_positive, indexes_negative)
self.split_by_ratio(self.test_split, indexes_positive,
indexes_negative)

indexes_train = indexes_positive + indexes_negative

Expand All @@ -134,12 +151,14 @@ def split_by_ratio(self, split_ratio, indexes_positive, indexes_negative):
"""Splits the indexes positive and negative by a ratio.

# Arguments
split_ratio: Float. Ratio of the split. Valid options are between 0.0 and 1.0.
split_ratio: Float. Ratio of the split. Valid options are
between 0.0 and 1.0.
indexes_positive: List of integers. Indexes of the positive samples.
indexes_negative: List of integers. Indexes of the negative samples.
"""
split_size = int(split_ratio * 0.5 * self.total_dataset_size)
indexes_split = indexes_positive[:split_size] + indexes_negative[:split_size]
indexes_split = (indexes_positive[:split_size] +
indexes_negative[:split_size])
indexes_positive = indexes_positive[split_size:]
indexes_negative = indexes_negative[split_size:]
return indexes_split, indexes_positive, indexes_negative
Expand All @@ -148,18 +167,22 @@ def reduction_init(self):
"""Initializes the reduction method.

# Arguments
reduction_method: String. Valid options are 'cut' or 'reduce'. If 'cut' is selected, the video is cut to the
reduction_length. If 'reduce' is selected, reduction_length many single frames of the video is removed
form the clip.
reduced_length: Float. The length of the video after the reduction_method is applied. Choose None if you
want to keep the original size. None is the default
reduction_method: String. Valid options are 'cut' or 'reduce'.
If 'cut' is selected, the video is cut to the reduction_length.
If 'reduce' is selected, reduction_length many single frames of
the video is removed form the clip.
reduced_length: Float. The length of the video after
the reduction_method is applied. Choose None if you want to keep
the original size. None is the default.
"""
if self.reduced_length is None:
self.reduced_length = self.video_length
elif self.reduced_length > self.video_length:
raise ValueError('reduction_length must be smaller than the length of the video')
raise ValueError('reduction_length must be smaller than the length '
'of the video')
else:
self.reduced_length = math.ceil(self.reduced_length / 25 * self.video_length)
self.reduced_length = math.ceil(self.reduced_length / 25 *
self.video_length)

if 'reduce' in self.reduction_method:
if self.reduced_length == self.video_length:
Expand All @@ -169,7 +192,8 @@ def reduction_init(self):

cal_drop_every = self.video_length / count_dropouts

self.dropout_ids = [int(i * cal_drop_every - (cal_drop_every / 2))
self.dropout_ids = [int(i * cal_drop_every -
(cal_drop_every / 2))
for i in range(1, count_dropouts + 1)]

def generate_x_data(self, x_train, i):
Expand All @@ -183,27 +207,32 @@ def generate_x_data(self, x_train, i):
if self.evaluating:
self.index.append(i)

if 'reduce' in self.reduction_method: # First tested using appending all wanted frames, but it is
# more efficient to remove the unwanted frames
if 'reduce' in self.reduction_method:
x_out = x_train[i]
x_out = np.delete(x_out, self.dropout_ids, 0)
else:
x_out = x_train[i][:self.reduced_length]
return x_out

def get_index(self):
"""Gets the index of the current sample. Only available if evaluating is set to True."""
"""Gets the index of the current sample. Only available if evaluating
is set to True.
"""
return self.index.pop(0)

def __len__(self):
if self.total_dataset_size == -1:
raise ValueError('You need to call __call__ first to set the total_size')
raise ValueError('You need to call __call__ first to set the '
'total_size')

if self.split == 'train':
return (self.total_dataset_size - int(self.validation_split * 0.5 * self.total_dataset_size) * 2 -
return (self.total_dataset_size -
int(self.validation_split * 0.5 *
self.total_dataset_size) * 2 -
int(self.test_split * 0.5 * self.total_dataset_size) * 2)
elif self.split == 'validation':
return int(self.validation_split * 0.5 * self.total_dataset_size) * 2
return int(self.validation_split * 0.5 *
self.total_dataset_size) * 2
elif self.split == 'test':
print('total_size_test', self.total_dataset_size)
return int(self.test_split * 0.5 * self.total_dataset_size) * 2
Loading