APPENDIX 4

Program Listing
 123

Receiver
// MultiTxAckPayload - the master or the transmitter
// works with two Arduinos as slaves
// one with the adress {'R','x','A','A','A'}
// and the other with {'R','x','A','A','B'}
//Libraries needed for NRF24L01
#include <SPI.h>
#include <nRF24L01.h>
#include <RF24.h>
int LED_GREEN = A3; //pin declaration for the LED
int LED_RED = A2;
#define CE_PIN 8
#define CSN_PIN 9
const byte numSlaves = 2;
const byte slaveAddress[numSlaves][5] = {{'R', 'x', 'A', 'A', 'A'},{'R', 'x', 'A', 'A', 'B'}};
RF24 radio(CE_PIN, CSN_PIN); // Create a Radio
char dataToSend[10] = "ToSlvN 0";
int ackData[10] = {0, 1, 2, 4, 5, 6, 7, 8, 9}; // to hold the sensor values coming from the slave
bool newData = false;
unsigned long currentMillis;
unsigned long prevMillis;
unsigned long txIntervalMillis = 80;
//=====================================
void setup() {
Serial.begin(19200);
radio.begin();
radio.setDataRate( RF24_250KBPS );
radio.setPALevel(RF24_PA_MIN);
radio.enableAckPayload();
radio.setRetries(4, 6); // delay, count
// radio.openWritingPipe(slaveAddress); -- moved to loop()
pinMode(LED_GREEN, OUTPUT); // pin mode for the led in the receiver
pinMode(LED_RED, OUTPUT);
}
//=============
void loop() {
currentMillis = millis();
if (currentMillis - prevMillis >= txIntervalMillis) {
Serial.print("1");
send();
}
}
//================
void send() {
// call each slave in turn
for (byte n = 0; n < numSlaves; n++) {
// open the writing pipe with the address of a slave
radio.openWritingPipe(slaveAddress[n]);
bool rslt;
rslt = radio.write( &dataToSend, sizeof(dataToSend) );
if (rslt) {
if ( radio.isAckPayloadAvailable() ) {
analogWrite(LED_GREEN, 1023);
analogWrite(LED_RED, 0);
radio.read(&ackData, sizeof(ackData));
newData = true;
}
else {
Serial.println(" Acknowledge but no data ");
}
 124

}
else {
Serial.print(",");
Serial.print(" 0, 0, 0, 0, 0, 0, 0, 0, 0, 0");
analogWrite(LED_RED, 1023);
analogWrite(LED_GREEN, 1023);
}
showData();
}
Serial.println();
prevMillis = millis();
}
//===============
void showData() {
if (newData == true) {
Serial.print(", ");
Serial.print(ackData[0]);
Serial.print(", ");
Serial.print(ackData[1]);
Serial.print(", ");
Serial.print(ackData[2]);
Serial.print(", ");
Serial.print(ackData[3]);
Serial.print(", ");
Serial.print(ackData[4]);
Serial.print(", ");
Serial.print(ackData[5]);
Serial.print(", ");
Serial.print(ackData[6]);
Serial.print(", ");
Serial.print(ackData[7]);
Serial.print(", ");
Serial.print(ackData[8]);
Serial.print(", ");
Serial.print(ackData[9]);
newData = false;
}
}
//================
Transmitter
//====MPU-6050 I2C LIBRARIES======
#include <Wire.h>
#include <I2Cdev.h>
#include <MPU6050.h>
//=====NRF24L01 SPI LIBRARIES =======
#include <SPI.h>
#include <nRF24L01.h>
#include <RF24.h>

#define CE_PIN 8
#define CSN_PIN 9
#define flex_1 A0 //pin assignment for the five
#define flex_2 A1 //flex sensors
#define flex_3 A2
#define flex_4 A3
#define flex_5 A6
int pinx = 2; // pin assignment for contact sensors
int piny = 3;
MPU6050 accelgyro;
int16_t ax, ay, az; //
 125

int16_t gx, gy, gz;

const byte thisSlaveAddress[5] = {'R', 'x', 'A', 'A', 'A'};
RF24 radio(CE_PIN, CSN_PIN);// create a radio
struct Pack2 {
int data1;
int data2;
int data3;
int data4;
int data5;
int accx;
int accy;
int accz;
int con1;
int con2;
} packet2;
uint16_t lastReport;
const int numReadings = 25;
#if defined(USE_ACCEL)
const int numAxis = USE_ACCEL;
const int AX = 0;
const int AY = 1;
const int AZ = 2;
#endif
int32_t readings[numAxis][numReadings]; // the reading history
int32_t readIndex[numAxis]; // the index of the current reading
int32_t total[numAxis]; // the running total
int32_t average[numAxis]; // the average
char dataReceived[10]; // this must match dataToSend in the TX
int ackData[10] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; // the sensor values to be sent to the master
bool newData = false;
//==============
void setup() {
lastReport = millis();
Serial.begin(19200);
Serial.println("SimpleRxAckPayload Starting");
radio.begin();
radio.setDataRate( RF24_250KBPS );
radio.openReadingPipe(1, thisSlaveAddress);
radio.enableAckPayload();
radio.startListening();
Wire.begin();
accelgyro.initialize();
accelgyro.setXAccelOffset(1000); //1000
accelgyro.setYAccelOffset(-101); //-101
accelgyro.setZAccelOffset(900); //900
for (int axis = 0; axis < numAxis; axis++) {
readIndex[axis] = 0;
total[axis] = 0;
average[axis] = 0;
for (int i = 0; i < numReadings; i++) {
readings[axis][i] = 0;
}
}
pinMode(pinx, INPUT);
digitalWrite(pinx, HIGH);
pinMode(piny, INPUT);
digitalWrite(piny, HIGH);
radio.writeAckPayload(1, &ackData, sizeof(ackData)); // pre-load data
}
//==========
 126

void loop() {
smooth(AX, ax);
smooth(AY, ay);
smooth(AZ, az);
packet2.data1 = analogRead(flex_1);
packet2.data1 = map(packet2.data1, 500, 900, 3000, 10000);
packet2.data2 = analogRead(flex_2);
packet2.data2 = map(packet2.data2, 500, 900, 1000, 10000);
packet2.data3 = analogRead(flex_3);
packet2.data3 = map(packet2.data3, 300, 500, 500, 10000);
packet2.data4 = analogRead(flex_4);
packet2.data4 = map(packet2.data4, 300, 500, 500, 10000);
packet2.data5 = analogRead(flex_5);
packet2.data5 = map(packet2.data5, 500, 900, 1000, 10000);
packet2.accx = average[AX];
packet2.accy = average[AY];
packet2.accz = average[AZ];
packet2.con1 = map(digitalRead(pinx), 0, 1, 1000, 2000);
packet2.con2 = map(digitalRead(piny), 0, 1, 1000, 2000);
accelgyro.getMotion6(&ax, &ay, &az, &gx, &gy, &gz);
getData();
showData();
}
//============
void getData() {
if ( radio.available() ) {
radio.read( &dataReceived, sizeof(dataReceived) );
updateReplyData();
newData = true;
}
}
//================
void showData() {
if (newData == true) {
Serial.print(ackData[0]);
Serial.print(", ");
Serial.print(ackData[1]);
Serial.print(", ");
Serial.print(ackData[2]);
Serial.print(", ");
Serial.print(ackData[3]);
Serial.print(", ");
Serial.print(ackData[4]);
Serial.print(", ");
Serial.print(ackData[5]);
Serial.print(", ");
Serial.print(ackData[6]);
Serial.print(", ");
Serial.print(ackData[7]);
Serial.print(", ");
Serial.print(ackData[8]);
Serial.print(", ");
Serial.println(ackData[9]);
newData = false;
}
}
//================
void updateReplyData() {
 127

ackData[0] = packet2.data1; ackData[1] = packet2.data2; ackData[2] = packet2.data3; ackData[3] =

packet2.data4; ackData[5] = packet2.accx; ackData[6] = packet2.accy; ackData[7] = packet2.accz;
ackData[8] = packet2.con1; ackData[9] = packet2.con2;

radio.writeAckPayload(1, &ackData, sizeof(ackData)); // load the payload for the next time
}
void smooth(int axis, int32_t val) {
total[axis] -= readings[axis][readIndex[axis]];
total[axis] += val;
readings[axis][readIndex[axis]] = val;
readIndex[axis]++;
if (readIndex[axis] >= numReadings)
readIndex[axis] = 0;
// calculate the average:
average[axis] = total[axis] / numReadings;
}

Python Main Program

# -*- coding: utf-8 -*-
from kivy.app import App
from kivy.lang import Builder
from kivy.uix.image import Image
from kivy.uix.behaviors import ButtonBehavior
from kivy.properties import BooleanProperty, ObjectProperty, NumericProperty, StringProperty,
ListProperty
from kivy.uix.progressbar import ProgressBar
from kivy.uix.scrollview import ScrollView
from kivy.core.window import Window
from kivy.graphics import Color, Rectangle
from kivy.uix.screenmanager import ScreenManager, Screen, SwapTransition
from kivy.uix.floatlayout import FloatLayout
from kivy.config import Config
from threading import Timer
import serial.tools.list_ports
import modeling
import imp
import numpy as np
import pandas as pd
import pyttsx
import sched
import time
import threading
import csv
import string
import collections
import serial
import random
import os, sys
Config.set('graphics', 'fullscreen', '0')
Builder.load_string("""

<MenuScreen>:
canvas.before:
Rectangle:
pos: self.pos
size: self.size
source: 'bgpics/blackboard.png'
FloatLayout:
Button:
text: "Learn"
 128

background_normal:'bgpics/chalk1.png'
background_down:'bgpics/chalk1.png'
color: 1, 1, 1, 1
size_hint: .2, .2
font_name: 'fonts/SqueakyChalkSound.ttf'
font_size: '40sp'
pos_hint:{"x":.09,"y":.35}
size_hint: .4, .4
on_press: root.manager.current = 'learncategories'
Button:
text: "Translate"
background_normal:'bgpics/chalk1.png'
background_down:'bgpics/chalk1.png'
color: 1, 1, 1, 1
font_name: 'fonts/SqueakyChalkSound.ttf'
font_size: '40sp'
pos_hint:{"x":.51,"y":.35}
size_hint: .4, .4
on_press: root.manager.current = 'translate'
Button:
text: "Quit"
background_normal:'bgpics/chalk2.png'
background_down:'bgpics/chalk2.png'
color: 1, 1, 1, 1
font_size: '20sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
pos_hint:{"x":.09,"y":.09}
size_hint: .2, .2
on_press:
app.get_running_app().stop()
Button:
text: "Gestures"
background_normal:'bgpics/chalk6.png'
background_down:'bgpics/chalk6.png'
color: 1, 1, 1, 1
font_size: '17sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
pos_hint:{"x":.49,"y":.13}
size_hint: .2, .2
on_press: root.manager.current = 'gestureslist'
Button:
text: "Play"
background_normal:'bgpics/chalk7.png'
background_down:'bgpics/chalk7.png'
color: 1, 1, 1, 1
font_size: '20sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
pos_hint:{"x":.29,"y":.13}
size_hint: .2, .2
on_press: root.manager.current = 'pregame'
Button:
text: "About"
background_normal:'bgpics/chalk9.png'
background_down:'bgpics/chalk9.png'
color: 1, 1, 1, 1
font_size: '20sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
pos_hint:{"x":.7,"y":.09}
size_hint: .2, .2
on_press: root.manager.current = 'aboutmenu'
 129

Label:
text: 'Filipino Sign Language Translator'
color: 1, 1, 1, 1
pos: -5, 217
font_size: '35sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
Label:
text: ' Cavite State University - Main Campus | Department of Computer and Electronics
Engineering '
halign: 'center'
color: 0, 0, 0, 1
pos_hint: {"x":.2, "y":-.46}
font_size: '11sp'
font_name: 'fonts/Futura.ttf'
<AddGestureScreen>:
canvas.before:
Rectangle:
pos: self.pos
size: self.size
source: 'bgpics/school2.png'
FloatLayout:
TextInput:
id: gesturename
color: 0, 0, 0, 1
font_name: 'fonts/FuturaHandwritten.ttf'
font_size: '30sp'
pos_hint:{"x":.2,"y":.5}
size_hint: .6, .1
multiline: False
write_tab: False
Button:
text: "Add Gesture"
background_normal:'bgpics/yepaint.png'
background_down:'bgpics/yepaint.png'
color: 0, 0, 0, 1
font_name: 'fonts/FuturaHandwritten.ttf'
font_size: '25sp'
pos_hint:{"x":.59,"y":.2}
size_hint: .25, .25
border:(16, 16, 16, 16)
keep_ratio: True
on_press:
root.manager.current = 'pregesture'
root.submit_gesturename()
Button:
text: "Back"
background_normal:'bgpics/yepaint.png'
background_down:'bgpics/yepaint.png'
color: 0, 0, 0, 1
font_size: '25sp'
font_name: 'fonts/FuturaHandwritten.ttf'
pos_hint:{"x":.19,"y":.2}
size_hint: .25, .25
on_press: root.manager.current = 'menu'
Label:
text: 'Add New Gesture'
color: 0, 0, 0, 1
pos: -5, 160
font_size: '35sp'
font_name: 'fonts/FuturaHandwritten.ttf'
 130

Label:
text: 'Type the name of new custom gesture below'
color: 0, 0, 0, 1
pos: -5, 100
font_size: '25sp'
font_name: 'fonts/FuturaHandwritten.ttf'

<PerformGestureScreen>:
canvas.before:
Rectangle:
pos: self.pos
size: self.size
source: 'bgpics/blackbg.png'
FloatLayout:
Image:
source: 'bgpics/load3.gif'
pos_hint:{"x":0.05,"y":.03}
size_hint: .9, .9
anim_delay: 0.05
allow_stretch: True
keep_ratio: False
Button:
text: "Back"
background_normal:'bgpics/white1.png'
background_down:'bgpics/white1.png'
color: 0, 0, 0, 1
font_size: '30sp'
font_name: 'fonts/FuturaHandwritten.ttf'
pos_hint:{"x":.05,"y":.05}
size_hint: .25, .25
on_press:
root.manager.current = 'addgesture'
root.quit_screen()
root.csvwriter()
ProgressBar:
id: pb
size_hint_x: 2
size_hint_y: None
pos_hint:{"x":.001,"y":.6}
height: '40dp'
max: 100
min:0
value: 0
<GesturesListScreen>:
canvas.before:
Rectangle:
pos: self.pos
size: self.size
source: 'bgpics/paperclip2.jpg'
FloatLayout:
ScrollView:
size_hint: (.4, .6)
pos_hint:{"x":.04,"y":.1}
bar_width: 8
bar_color: 1, 0, 0, 1 # red
bar_inactive_color: 0, 0, 1, 1 # blue
effect_cls: "ScrollEffect"
scroll_type: ['bars']
Label:
id: defaultgestureskv
 131

text: ""
font_size: 20
color: 0, 0, 0, 1
text_size: self.width, None
halign: "center"
size_hint_y: None
height: self.texture_size[1]
ScrollView:
size_hint: (.3, .6)
pos_hint:{"x":.4,"y":.1}
bar_width: 8
bar_color: 1, 0, 0, 1 # red
bar_inactive_color: 0, 0, 1, 1 # blue
effect_cls: "ScrollEffect"
scroll_type: ['bars']
Label:
id: customgestureskv
text: ""
font_size: 20
color: 0, 0, 0, 1
text_size: self.width, None
halign: "center"
size_hint_y: None
height: self.texture_size[1]
FloatLayout:
Label:
text:"DEFAULT GESTURES:"
font_size: 20
color: 0, 0, 0, 1
font_name: 'fonts/FuturaHandwritten.ttf'
font_size: '20sp'
pos_hint:{"x":-.26,"y":.25}
Label:
id: gesturescount
text:""
font_size: 30
color: 1, 0, 0, 1
font_name: 'fonts/FuturaHandwritten.ttf'
font_size: '35sp'
pos_hint:{"x":-.115,"y":.255}
Label:
text:"CUSTOM GESTURES:"
font_size: 20
color: 0, 0, 0, 1
font_name: 'fonts/FuturaHandwritten.ttf'
font_size: '20sp'
pos_hint:{"x":.07,"y":.25}
Label:
id: gesturescount2
text:""
font_size: 30
color: 1, 0, 0, 1
font_name: 'fonts/FuturaHandwritten.ttf'
font_size: '35sp'
pos_hint:{"x":.2,"y":.255}
Button:
text: "Delete Gesture"
background_normal:'bgpics/brush4.png'
background_down:'bgpics/brush4.png'
color: 0, 0, 0, 1
 132

font_name: 'fonts/FuturaHandwritten.ttf'
font_size: '20sp'
pos_hint:{"x":.72,"y":.4}
size_hint: .18, .20
on_press:
root.manager.current = 'deletegesture'
Button:
text: "Add Gesture"
background_normal:'bgpics/brush4.png'
background_down:'bgpics/brush4.png'
color: 0, 0, 0, 1
font_name: 'fonts/FuturaHandwritten.ttf'
font_size: '20sp'
pos_hint:{"x":.72,"y":.6}
size_hint: .18, .20
on_press:
root.manager.current = 'addgesture'
Button:
text: "BACK"
background_normal:'bgpics/brush4.png'
background_down:'bgpics/brush4.png'
color: 0, 0, 0, 1
font_size: '20sp'
font_name: 'fonts/FuturaHandwritten.ttf'
pos_hint:{"x":.72,"y":.2}
size_hint: .18, .20
on_press: root.manager.current = 'menu'
Label:
text: 'Gestures List'
background_normal:'bgpics/yepaint.png'
background_down:'bgpics/yepaint.png'
color: 0, 0, 0, 1
pos: -5, 220
font_size: '50sp'
font_name: 'fonts/FuturaHandwritten.ttf'
Image:
source: 'bgpics/plus.png'
pos_hint:{"x":.775,"y":.715}
size_hint: .065, .065
Image:
source: 'bgpics/minus.png'
pos_hint:{"x":.780,"y":.525}
size_hint: .055, .055
Image:
source: 'bgpics/go-back.png'
pos_hint:{"x":.780,"y":.325}
size_hint: .055, .055

<LearnScreen>:
canvas.before:
Rectangle:
pos: self.pos
size: self.size
source: 'bgpics/categorybg.jpg'
FloatLayout:
id: learn_screen_layout
Button:
text: "Alphabet"
background_normal:'bgpics/ppr.png'
background_down:'bgpics/ppr.png'
 133

color: 0, 0, 0, 1
size_hint: .4, .4
font_name: 'fonts/DK Lemon Yellow Sun.otf'
font_size: '50sp'
pos_hint:{"x":.09,"y":.3}
on_press: root.manager.current = 'alphapop1'
Button:
text: "Words/Phrases"
background_normal:'bgpics/sticky2.png'
background_down:'bgpics/sticky2.png'
color: 0, 0, 0, 1
font_name: 'fonts/DK Lemon Yellow Sun.otf'
font_size: '50sp'
pos_hint:{"x":.51,"y":.3}
size_hint: .4, .4
on_press: root.manager.current = 'wordpop1'
Button:
text: "BACK"
background_normal:'bgpics/yepaint.png'
background_down:'bgpics/yepaint.png'
color: 0, 0, 0, 1
font_size: '35sp'
font_name: 'fonts/DK Lemon Yellow Sun.otf'
pos: 50, 5
size_hint: .27, .27
on_press: root.manager.current = 'menu'
Label:
id: loki
text: 'Categories'
background_normal:'bgpics/yepaint.png'
color: 0, 0, 0, 1
pos: -5, 210
font_size: '60sp'
font_name: 'fonts/DK Lemon Yellow Sun.otf'
<AlphaPop1Screen>:
canvas.before:
Rectangle:
pos: self.pos
size: self.size
source: 'bgpics/translateback.jpg'
FloatLayout:
Label:
text: 'Perform the shown gestures correctly.'
background_normal:'bgpics/chalk2.png'
color: 0, 0, 0, 1
pos: -5, 140
font_size: '25sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
Label:
text: 'Use the NEXT and BACK buttons to'
background_normal:'bgpics/chalk2.png'
color: 0, 0, 0, 1
pos: -5, 90
font_size: '25sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
Label:
text: 'navigate between the letters of the alphabet.'
background_normal:'bgpics/chalk2.png'
color: 0, 0, 0, 1
pos: -5, 40
 134

font_size: '25sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
Button:
text: "BEGIN"
background_normal:'bgpics/purp1.png'
background_down:'bgpics/purp1.png'
color: 1, 1, 0, 1
size_hint: .6, .3
font_name: 'fonts/DK Lemon Yellow Sun.otf'
font_size: '70sp'
pos_hint:{"x":.2,"y":.1}
on_press: root.manager.current = 'letterlearn'
Image:
source: 'bgpics/paperclip.png'
pos_hint:{"x":.05,"y":.02}
size_hint: .2, .2
<AlphaPopEndScreen>:
canvas.before:
Rectangle:
pos: self.pos
size: self.size
source: 'bgpics/pastel2.jpg'
FloatLayout:
Label:
text: 'Nice one!'
background_normal:'bgpics/chalk2.png'
background_down:'bgpics/chalk2.png'
color: 0, 0, 0, 1
pos: -5, 200
font_size: '35sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
Label:
text: 'You gestured the 26 letters'
background_normal:'bgpics/chalk2.png'
color: 0, 0, 0, 1
pos: -5, 140
font_size: '35sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
Label:
text: 'of the alphabet!'
background_normal:'bgpics/chalk2.png'
color: 0, 0, 0, 1
pos: -5, 90
font_size: '35sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
Label:
text: 'GREAT JOB!'
background_normal:'bgpics/chalk2.png'
color: 0, 0, 0, 1
pos: -5, 10
font_size: '35sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
Button:
text: "RESTART"
background_normal:'bgpics/white1.png'
color: 0, 0, 0, 1
size_hint: .4, .2
font_name: 'fonts/DK Lemon Yellow Sun.otf'
font_size: '65sp'
pos_hint:{"x":.05,"y":.1}
 135

on_press:
root.manager.current = 'letterlearn'
Button:
text: "QUIT"
background_normal:'bgpics/white1.png'
background_down:'bgpics/white1.png'
color: 0, 0, 0, 1
size_hint: .4, .2
font_name: 'fonts/DK Lemon Yellow Sun.otf'
font_size: '65sp'
pos_hint:{"x":.5,"y":.1}
on_press:
root.manager.current = 'menu'
Image:
source: 'bgpics/crayon-balloon-1.png'
pos_hint:{"x":.01,"y":.7}
size_hint: .2, .2
Image:
source: 'bgpics/crayon-balloon-2.png'
pos_hint:{"x":.8,"y":.6}
size_hint: .2, .2
Image:
source: 'bgpics/crayon-balloon-3.png'
pos_hint:{"x":.09,"y":.4}
size_hint: .2, .2
Image:
source: 'bgpics/crayon-balloon-4.png'
pos_hint:{"x":.7,"y":.4}
size_hint: .2, .2

<LearnLetterScreen>:
canvas.before:
Rectangle:
pos: self.pos
size: self.size
source: 'bgpics/categorybg.jpg'
FloatLayout:
id: correct_layout
opacity: 0
Label:
text: 'Gesture Correct!'
background_normal:'bgpics/translateback.jpg'
pos_hint:{"x":-0.15,"y":-.43}
color: 1,1,0,1,1
font_size: '45sp'
font_name: 'fonts/EraserRegular.ttf'
Image:
source: 'bgpics/check2.png'
pos_hint:{"x":0.66,"y":.03}
size_hint: .1, .1
allow_stretch: True
keep_ratio: False
FloatLayout:
id: correct_pic
opacity: 0.1
Image:
source: root.letterfilename
pos_hint:{"x":0.43,"y":.15}
size_hint: .40, .70
allow_stretch: True
 136

keep_ratio: False
FloatLayout:
id: orig_pic
Image:
source: root.letterfilename
pos_hint:{"x":0.009,"y":.15}
size_hint: .40, .70
allow_stretch: True
keep_ratio: False
FloatLayout:
id: try_again
opacity: 0
Label:
text: 'Try Again!'
background_normal:'bgpics/translateback.jpg'
pos_hint:{"x":-0.15,"y":-.43}
color: 0,1,0,1,1
font_size: '45sp'
font_name: 'fonts/EraserRegular.ttf'
FloatLayout:
id: do_ges
opacity: 0
Label:
text: 'Do the gesture in the picture.'
background_normal:'bgpics/translateback.jpg'
pos_hint:{"x":-0.10,"y":-.43}
font_size: '33sp'
font_name: 'fonts/EraserRegular.ttf'
color: 0, 0, 0, 1
FloatLayout:
Label:
text: 'Learning the FSL Alphabet'
background_normal:'bgpics/chalk2.png'
pos_hint:{"x":0.009,"y":.43}
color: 0, 0, 0, 1
font_size: '45sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
Button:
text: "NEXT"
background_normal:'bgpics/translateback.jpg'
background_down:'bgpics/translateback.jpg'
font_size: '35sp'
font_name: 'fonts/vtks.ttf'
color: 0, 0, 0, 1
pos_hint:{"x":.87,"y":.6}
size_hint: .1, .1
on_press:
root.next_screen()
on_release:
correct_pic.source=root.letterfilename
orig_pic.source=root.letterfilename
Button:
text: "BACK"
background_normal:'bgpics/translateback.jpg'
background_down:'bgpics/translateback.jpg'
font_size: '35sp'
font_name: 'fonts/vtks.ttf'
color: 0, 0, 0, 1
pos_hint:{"x":.87,"y":.4}
size_hint: .1, .1
 137

on_press:
root.prev_screen()
on_release:
correct_pic.source=root.letterfilename
orig_pic.source=root.letterfilename
Button:
text: "QUIT"
background_normal:'bgpics/translateback.jpg'
background_down:'bgpics/translateback.jpg'
font_size: '35sp'
font_name: 'fonts/vtks.ttf'
color: 0, 0, 0, 1
pos_hint:{"x":.87,"y":.2}
size_hint: .1, .1
on_press:
root.manager.current = 'menu'
root.stop_callback()
<LearnWordScreen>:
canvas.before:
Rectangle:
pos: self.pos
size: self.size
source: 'bgpics/categorybg.jpg'
FloatLayout:
id: correct_layout
opacity: 0
Label:
text: 'Gesture Correct!'
background_normal:'bgpics/translateback.jpg'
pos_hint:{"x":-0.15,"y":-.43}
color: 1,1,0,1,1
font_size: '45sp'
font_name: 'fonts/EraserRegular.ttf'
Image:
source: 'bgpics/check2.png'
pos_hint:{"x":0.66,"y":.03}
size_hint: .1, .1
allow_stretch: True
keep_ratio: False
FloatLayout:
id: correct_pic
opacity: 0.1
Image:
source: root.wordfilename
pos_hint:{"x":0.43,"y":.15}
size_hint: .35, .60
anim_delay: 0.05
allow_stretch: True
keep_ratio: False
FloatLayout:
id: orig_pic
Image:
source: root.wordfilename
pos_hint:{"x":0.05,"y":.15}
size_hint: .35, .60
anim_delay: 0.05
allow_stretch: True
keep_ratio: False
FloatLayout:
Label:
 138

id: words_name2
text: ""
pos_hint:{"x":0.1,"y":.2}
color: 0,0,0,1
font_size: '53sp'
font_name: 'fonts/EraserRegular.ttf'
FloatLayout:
Label:
id: words_name
text: ""
pos_hint:{"x":-0.28,"y":.2}
color: 0,0,0,1
font_size: '53sp'
font_name: 'fonts/EraserRegular.ttf'
FloatLayout:
id: try_again
opacity: 0
Label:
text: 'Try Again!'
background_normal:'bgpics/translateback.jpg'
pos_hint:{"x":-0.15,"y":-.43}
color: 0,1,0,1,1
font_size: '45sp'
font_name: 'fonts/EraserRegular.ttf'
FloatLayout:
id: do_ges
opacity: 0
Label:
text: 'Do the gesture as shown.'
background_normal:'bgpics/translateback.jpg'
pos_hint:{"x":-0.10,"y":-.43}
font_size: '33sp'
font_name: 'fonts/EraserRegular.ttf'
color: 0, 0, 0, 1
FloatLayout:
Label:
text: 'Learning the FSL\\n Basic Words/Phrases'
valign: 'center'
halign: 'center'
background_normal:'bgpics/chalk2.png'
color: 0 ,0 ,0, 1
pos_hint:{"x":0.009,"y":.37}
font_size: '45sp'
font_name: 'fonts/SqueakyChalkSound.ttf'
Button:
text: "NEXT"
background_normal:'bgpics/translateback.jpg'
background_down:'bgpics/translateback.jpg'
font_size: '35sp'
font_name: 'fonts/vtks.ttf'
color: 0, 0, 0, 1
pos_hint:{"x":.87,"y":.6}
size_hint: .1, .1
on_press:
root.next_screen()
on_release:
correct_pic.source=root.wordfilename
orig_pic.source=root.wordfilename
Button:
text: "BACK"
 139

background_normal:'bgpics/translateback.jpg'
background_down:'bgpics/translateback.jpg'
font_size: '35sp'
font_name: 'fonts/vtks.ttf'
color: 0, 0, 0, 1
pos_hint:{"x":.87,"y":.4}
size_hint: .1, .1
on_press:
root.prev_screen()
on_release:
correct_pic.source=root.wordfilename
orig_pic.source=root.wordfilename
Button:
text: "QUIT"
background_normal:'bgpics/translateback.jpg'
background_down:'bgpics/translateback.jpg'
font_size: '35sp'
font_name: 'fonts/vtks.ttf'
color: 0, 0, 0, 1
pos_hint:{"x":.87,"y":.2}
size_hint: .1, .1
on_press:
root.manager.current = 'menu'
root.stop_callback()
<TranslateScreen>:
canvas.before:
Rectangle:
pos: self.pos
size: self.size
source: 'bgpics/sheet.png'
FloatLayout:
Label:
text: 'FSL Translator'
background_normal:'bgpics/chalk2.png'
pos_hint:{"x":0.009,"y":.35}
font_size: '45sp'
color: 0, 0, 0, 1
font_name: 'fonts/Stay.otf'
Button:
background_normal:'bgpics/orpaint.png'
background_down:'bgpics/orpaint.png'
text: 'BACK'
color: 0, 0, 0, 1
font_size: '35sp'
font_name: 'fonts/vtks.ttf'
pos: 60, 35
size_hint: .2, .2
on_press:
root.quit_screen()
root.manager.current = 'menu'
FloatLayout:
id: letter
Label:
text: root.transindexname
color: 0, 0, 0, 1
pos: -5, 70
font_size: '200sp'
font_name: 'fonts/REIS.otf'

<GestureGameScreen>:
 140

canvas.before:
Rectangle:
pos: self.pos
size: self.size
source: 'bgpics/game_bgfinal.png'
FloatLayout:
Image:
id:questionbg
opacity:0
source: 'bgpics/game_question.png'
pos:self.pos
size: self.size
allow_stretch: True
keep_ratio: False
Image:
id:correctbg
opacity:0
source: 'bgpics/game_correct.png'
pos:self.pos
size: self.size
allow_stretch: True
keep_ratio: False
Image:
id:wrongbg
opacity:0
source: 'bgpics/game_wrong.png'
pos:self.pos
size: self.size
allow_stretch: True
keep_ratio: False
Image:
id: correct_image
opacity: 0
source: root.imageaddress
pos_hint:{"x":.15,"y":.13}
size_hint: .30, .45
allow_stretch: True
keep_ratio: False
Label:
id: questiongesture
text: ""
color: 0, 0, 0, 1
pos_hint:{"x":-.02,"y":.15}
font_size: '90sp'
font_name: 'fonts/FuturaHandwritten.ttf'
Label:
id: scoreboard
text: ""
color: 0, 0, 0, 1
pos_hint:{"x":-.36,"y":.32}
font_size: '35sp'
font_name: 'fonts/FuturaHandwritten.ttf'
Label:
id: questioncount
text: ""
color: 0, 0, 0, 1
pos_hint:{"x":.34,"y":.32}
font_size: '35sp'
font_name: 'fonts/FuturaHandwritten.ttf'
Label:
 141

id: userdidthis
text: ""
color: 0, 0, 1, 0.8
pos_hint:{"x":.0005,"y":-.10}
font_size: '50sp'
font_name: 'fonts/Funsized.ttf'
Button:
text: " "
font_size: '35sp'
font_name: 'fonts/vtks.ttf'
background_color: 0, 0, 0, 0
pos_hint:{"x":.79,"y":.13}
size_hint: .1, .1
on_press:
root.manager.current = 'menu'
root.serial_stop()
<AboutMenuScreen>:
canvas.before:
Rectangle:
pos: self.pos
size: self.size
source: 'bgpics/translatebg.png'
FloatLayout:
Button:
id: about_back
text: "Back"
background_normal:'bgpics/yepaint.png'
background_down:'bgpics/yepaint.png'
color: 0, 0, 0, 1
font_size: '25sp'
font_name: 'fonts/FuturaHandwritten.ttf'
bold: True
pos_hint:{"x":.42,"y":.03}
size_hint: .15, .15
on_press:
root.manager.current = 'menu'
Button:
id: dev_header
text: "About the Developers"
background_normal:'bgpics/IDEA.png'
background_down:'bgpics/IDEA.png'
color: 0, 0, 0, 1
font_size: '23sp'
font_name: 'fonts/Funsized.ttf'
bold: True
pos_hint:{"x":.22,"y":.83}
size_hint: .11, .15
on_press:
root.display_dev()
self.color = (1.0, 0.0, 0.0, 1.0)
Button:
id: app_header
text: "About the Device"
background_normal:'bgpics/HAND.png'
background_down:'bgpics/HAND.png'
color: 0, 0, 0, 1
font_size: '23sp'
font_name: 'fonts/Funsized.ttf'
bold: True
pos_hint:{"x":.66,"y":.83}
 142

size_hint: .11, .15

on_press:
root.display_app()
self.color = (1.0, 0.0, 0.0, 1.0)
FloatLayout:
id: dev_layout
Image:
source: 'bgpics/nino1.png'
allow_stretch: True
keep_ratio: True
pos_hint:{"x":.08,"y":.57}
size_hint: .25, .25
Image:
source: 'bgpics/ADRIAN.png'
allow_stretch: True
keep_ratio: True
pos_hint:{"x":.08,"y":.30}
size_hint: .25, .25
Label:
text: 'Marco Nino Oronica’
halign: 'left'
color: 0, 0, 0, 1
pos_hint: {"x":.12,"y":.2}
font_size: '15sp'
font_name: 'fonts/FuturaHandwritten.ttf'
Label:
text: 'Adrian Miguel J. Catibayan’
halign: 'left'
color: 0, 0, 0, 1
pos_hint: {"x":.10,"y":-.1}
font_size: '15sp'
font_name: 'fonts/FuturaHandwritten.ttf'
FloatLayout:
id: app_layout
Label:
text: 'The Filipino Sign Language Translator was created by developers Niño and Adrian\\n
aiming to provide an accessible, interactive educational platform for learning\\n fingerspelling and
basic Filipino Sign Language (FSL) to the Filipino youth,\\n may they be deaf and mute, or abled. It
may also serve as an alternative communication means,\\n bridging the communication gap between
the deaf/mute youth and the abled youth.\\n It features the following modes:\\n
halign: 'center'
valign: 'center'
color: 0, 0, 0, 1
pos_hint: {"x":.01,"y":.08}
font_size: '16sp'
font_name: 'fonts/FuturaHandwritten.ttf'
Image:
source: 'bgpics/cvsu_logo.png'
allow_stretch: False
keep_ratio: False
pos_hint:{"x":.32,"y":.12}
size_hint: .155, .25
Image:
source: 'bgpics/COESSLOGO.png'
allow_stretch: False
keep_ratio: False
pos_hint:{"x":.55,"y":.12}
size_hint: .1, .25
Image:
source: 'bgpics/CEIT.png'
 143

allow_stretch: False
keep_ratio: False
pos_hint:{"x":.45,"y":.12}
size_hint: .1, .25
<GameScoreScreen>:
canvas.before:
Rectangle:
pos: self.pos
size: self.size
source: 'bgpics/gamescorebg.png'
FloatLayout:
Label:
id: finalscore
text: ""
color: 0, 0, 0, 1
pos_hint:{"x":-.02,"y":.07}
font_size: '60sp'
font_name: 'fonts/ObelixPro.ttf'
Label:
text: "Congratulations!\\nYour csv_for_translate score is"
color: 0, 0, 0, 1
pos_hint:{"x":.009,"y":.25}
font_size: '50sp'
font_name: 'fonts/ObelixPro.ttf'
Button:
text: "Thank You!"
background_normal:'bgpics/quote1.png'
background_down:'bgpics/quote1.png'
color: 0, 0, 0, 1
font_name: 'fonts/FuturaHandwritten.ttf'
font_size: '25sp'
pos_hint:{"x":.18,"y":.2}
size_hint: .3, .3
on_press: root.manager.current = 'menu'
Button:
text: "Play Again!"
background_normal:'bgpics/quote1.png'
background_down:'bgpics/quote1.png'
color: 0, 0, 0, 1
font_name: 'fonts/FuturaHandwritten.ttf'
font_size: '25sp'
pos_hint:{"x":.48,"y":.2}
size_hint: .3, .3
on_press: root.manager.current = 'gesturegame'
""")
triviadictionary=['Two movement of the same handshape\ncan have separate meanings.',
'Babies can communicate using sign language\n6-8 months prior to communicating
verbally.',
'One sign in sign language can have\nmultiple meanings','Different countries have different
Sign Language',

class TriviaScreen(Screen):
triviaid=StringProperty("")
def on_enter(self):
trivia=random.choice(triviadictionary)
self.ids['triviaid'].text=str(trivia)
#Declare Screens
class MenuScreen(Screen):
pass
class LearnLetterScreen(Screen):
 144

letterfilename=StringProperty("")
def on_enter(self):
from modeling_letter import model
ports = list(serial.tools.list_ports.comports())
for p in ports:
comvar = str(p)
print p
if ('USB-SERIAL CH340') in comvar:
comportvar=comvar.split(' ', 1)[0]
break
print comportvar
self.comportvar=comportvar
alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
self.alphabet=alphabet
c = threading.Condition()
self.c=c
global flag
global num
global lock
lock=1
num=1
flag = 0
initial_letter='A'
current_letter=initial_letter
self.current_letter=current_letter
letterfilename=StringProperty("")
letterfilename='handgesture/a.png'
self.letterfilename=letterfilename
self.ids.do_ges.opacity = 1
self.thread1=threading.Thread(target=self.serialwriter)
self.thread1.start()
self.thread2=threading.Thread(target=self.checker)
self.thread2.start()
def next_screen(self, *args):
letterfilename=StringProperty("")
for i in range(len(self.alphabet)):
if(self.alphabet[i] == self.current_letter):
if(self.current_letter == 'Z'):
self.parent.current = 'alphapopend'
global num
global lock
lock=0
num=0
else:
i=i+1
print self.alphabet[i]
self.next_letter=self.alphabet[i]
self.current_letter=self.alphabet[i]
self.letterfilename=str('handgesture/'+self.current_letter+'.png')
#print self.letterfilename
break
lock=1
num=1
self.ids.do_ges.opacity = 1
self.ids.correct_layout.opacity = 0
self.ids.correct_pic.opacity = 0.1
self.ids.try_again.opacity = 0
self.thread1=threading.Thread(target=self.serialwriter)
self.thread1.start()
self.thread2=threading.Thread(target=self.checker)
 145

self.thread2.start()

def prev_screen(self):
letterfilename=StringProperty("")
for i in range(len(self.alphabet)):
if(self.alphabet[i] == self.current_letter and (self.current_letter !='a')):
i=i-1
print self.alphabet[i]
self.prev_letter=self.alphabet[i]
self.current_letter=self.alphabet[i]
self.letterfilename=str('handgesture/'+self.current_letter+'.png')
#print self.letterfilename
break
lock=1
num=1
self.ids.do_ges.opacity = 1
self.ids.correct_layout.opacity = 0
self.ids.correct_pic.opacity = 0.1
self.ids.try_again.opacity = 0
self.thread1=threading.Thread(target=self.serialwriter)
self.thread1.start()
self.thread2=threading.Thread(target=self.checker)
self.thread2.start()

def serialwriter(self, *args):

from modeling_letter import model
global lock
global num
global flag
while (num>0):
ser = serial.Serial(self.comportvar, 19200)
predicted_svm="something"
while True:
count = 0
data = []
while (count < 6):
# time.sleep(1)
if (ser.isOpen() == False):
ser.open()
ser.flushInput()
#print(count)
elif (count == 5):
data.append(ser.readline())
ser.close()
else:
data.append(ser.readline())
count = count + 1
data1 = data
data1.pop(0)
lines = []
for line in data1:
lines.append(line.decode('utf-8', 'ignore'))
lines1 = lines
[s.strip('\r\n') for s in lines1]
y = []
for x in lines1:
y.append(map(int, x.split(",")))
if(int(y[0][10])== 0):
predicted_svm="nothing"
print("nothing")
 146

else:
for i in range(len(y)):
y[i].pop(0)
test = model.predict(y)
m = [item for item, count in collections.Counter(test).items() if count > 2]
if(m==[]):
print 'nothing'
else:
predvar=m
self.c.acquire()
if (flag == 0):
predvar=m[0]
self.predvar=predvar
flag = 1
print "serial sent"
print self.predvar
print self.current_letter
self.c.notify_all()
else:
self.c.wait()
self.c.release()
if (lock==0 and num==0):
break
if (lock==0 and num==0):
break
def checker(self,*args):
global flag
global lock
global num
while (lock==1 and num==1):
self.c.acquire()
if flag == 1:
if (self.predvar == self.current_letter):
self.ids.correct_layout.opacity = 1
self.ids.correct_pic.opacity = 1
self.ids.try_again.opacity = 0
self.ids.do_ges.opacity = 0
engine = pyttsx.init()
rate = engine.getProperty('rate') # getting details of current speaking rate
engine.setProperty('rate', 250) # setting up new voice rate
engine.say(self.predvar)
engine.runAndWait()
lock=0
num=0
Clock.schedule_once(self.next_screen, 1.5)
else:
self.ids.try_again.opacity = 1
self.ids.do_ges.opacity = 0
print self.predvar
print self.current_letter
print "received"
flag=0
print "serial received"
self.c.notify_all()
else:
self.c.wait()
self.c.release()
if (lock==0 and num==0):
break
 147

class LearnWordScreen(Screen):
wordfilename=StringProperty("")
def on_enter(self):
#imp.reload(sys.modules['modeling_words'])
from modeling_words import model
ports = list(serial.tools.list_ports.comports())
for p in ports:
comvar = str(p)
print p
if ('USB-SERIAL CH340') in comvar:
comportvar=comvar.split(' ', 1)[0]
break
self.comportvar=comportvar
word = pd.read_csv('words.csv')
word = word['WORDS'].tolist()
self.word=word
c = threading.Condition()
self.c=c
global flag4
global num4
global lock4
lock4=1
num4=1
flag4 = 0
initial_word= 'CHURCH'
current_word=initial_wor
self.current_word=current_word
self.ids['words_name'].text=str(self.current_word)
wordfilename=StringProperty("")
wordfilename='handgesture/church.gif'
self.wordfilename=wordfilename
self.ids.do_ges.opacity = 1
self.thread1=threading.Thread(target=self.serialwriter)
self.thread1.start()
self.thread2=threading.Thread(target=self.checker)
self.thread2.start()
def next_screen(self, *args):
wordfilename=StringProperty("")
for i in range(len(self.word)):
if(self.word[i] == self.current_word):
if(self.current_word == 'YES'):
global lock4
global num4
global flag4
lock4=0
num4=0
flag4=0
self.parent.current = 'wordpopend'
else:
i=i+1
print self.word[i]
self.next_word=self.word[i]
self.current_word=self.word[i]
self.ids['words_name'].text=str(self.current_word)
self.ids['words_name2'].text=str("")
self.wordfilename=str('handgesture/'+self.current_word+'.gif')
print self.wordfilename
break

lock4=1
 148

num4=1
self.ids.do_ges.opacity = 1
self.ids.correct_layout.opacity = 0
self.ids.correct_pic.opacity = 0.1
self.ids['words_name'].text=str(self.current_word)
self.ids.try_again.opacity = 0
self.thread1=threading.Thread(target=self.serialwriter)
self.thread1.start()
self.thread2=threading.Thread(target=self.checker)
self.thread2.start()
def prev_screen(self):
wordfilename=StringProperty("")
for i in range(len(self.word)):
if(self.word[i] == self.current_word and (self.current_word !='CHURCH')):
i=i-1
print self.word[i]
self.prev_word=self.word[i]
self.current_word=self.word[i]
self.ids['words_name'].text=str(self.current_word)
self.ids['words_name2'].text=str("")
self.wordfilename=str('handgesture/'+self.current_word+'.gif')
print self.wordfilename
break
global num4
global lock4
lock4=1
num4=1
self.ids.do_ges.opacity = 1
self.ids.correct_layout.opacity = 0
self.ids.correct_pic.opacity = 0.1
self.ids['words_name'].text=str(self.current_word)
self.ids.try_again.opacity = 0
self.thread1=threading.Thread(target=self.serialwriter)
self.thread1.start()
self.thread2=threading.Thread(target=self.checker)
self.thread2.start()

def checker(self,*args):
global flag4
global lock4
global num4
while (lock4==1 and num4==1):
self.c.acquire()
if flag4 == 1:
if (self.predvar == self.current_word):
self.ids['words_name2'].text=str(self.current_word)
self.ids.correct_layout.opacity = 1
self.ids.correct_pic.opacity = 1
self.ids.try_again.opacity = 0
self.ids.do_ges.opacity = 0
engine = pyttsx.init()
rate = engine.getProperty('rate') # getting details of current speaking rate
engine.setProperty('rate', 200) # setting up new voice rate
engine.say(self.predvar)
engine.runAndWait()
lock4=0
num4=0
Clock.schedule_once(self.next_screen, 1)
else:
self.ids.try_again.opacity = 1
 149

self.ids.do_ges.opacity = 0
flag4=0
self.c.notify_all()
else:
self.c.wait()
self.c.release()
if (lock4==0 and num4==0):
break

class TranslateScreen(Screen):
transindexname=StringProperty("")
flag = 0
def on_enter(self):
imp.reload(sys.modules['modeling'])
ports = list(serial.tools.list_ports.comports())
for p in ports:
comvar = str(p)
print p
if ('USB-SERIAL CH340') in comvar:
comportvar=comvar.split(' ', 1)[0]
break
print comportvar
self.comportvar=comportvar
transindexname=StringProperty("")
c = threading.Condition()
self.c=c
global flag
global num1
global lock1
lock=1
num=1
flag = 0
self.thread1=threading.Thread(target=self.serialtest)
self.thread1.start()
self.thread2=threading.Thread(target=self.submit_gesturename)
self.thread2.start()
def submit_gesturename(self):
global flag
global transindexname
while True:
self.c.acquire()
if (flag == 1):
transindexname=StringProperty()
serialtranslabel=self.predvar
self.transindexname=str(serialtranslabel)
engine = pyttsx.init()
rate = engine.getProperty('rate') # getting details of current speaking rate
engine.setProperty('rate', 230) # setting up new voice rate
engine.say(serialtranslabel)
engine.runAndWait()
flag = 0
self.c.notify_all()
else:
self.c.wait()
self.c.release()
if (lock1==0 and num1==0):
break
def quit_screen(self):
self.serial_stop()
 150

class AddGestureScreen(Screen):
def submit_gesturename(self):
global indexname
indexname=self.ids.gesturename.text
indexname = indexname.upper()
customgestures=open('customgestureslist.csv','a')
customgestures.write("\n"+indexname)
def on_leave(self):
self.ids['gesturename'].text =str("")

class CountdownScreen(Screen):
def on_enter(self):
self.num=3
Clock.schedule_interval(self.counter, 1)
def counter(self, *args):
while (self.num>0):
self.ids['thecountdown'].text=str(self.num)
self.num=self.num-1
break
if (self.num==0):
Clock.unschedule(self.counter)
self.parent.current='performgesture'

class PerformGestureScreen(Screen):
def on_enter(self):
ports = list(serial.tools.list_ports.comports())
for p in ports:
comvar = str(p)
print p
if ('USB-SERIAL CH340') in comvar:
comportvar=comvar.split(' ', 1)[0]
break
self.comportvar=comportvar
self.thread1=threading.Thread(target=self.serialadd)
self.thread1.start()
Clock.schedule_interval(self.loading, 1)
self.ids.pb.value=0
self.n=0
def loading(self, *args):
while (self.n<=70):
self.ids.pb.value=self.n
self.n+=10
break
if (self.n==70):
Clock.unschedule(self.loading)
self.csvwriter()
def quit_screen(self, *args):
self.ids.pb.value=0
self.n=0
Clock.unschedule(self.loading)
def serialadd(self):
ser = serial.Serial(self.comportvar, baudrate=19200)
ser.flushInput()
first =True
count = 0
while True:
while (count < 30):
ser_bytes = ser.readline()
print(ser_bytes)
if first == True:
 151

time.sleep(0.1)
first = False
else:
file = open("csv_for_translate.csv", "a")
file.write(str(ser_bytes))
file.close()
count = count + 1
ser.close
ser.close()
def csvwriter(self):
global indexname
gesturename=indexname
df= pd.read_csv("csv_for_translate.csv")
find = ['1']
replace = [gesturename]
col = 'LABEL'
df[col] = df[col].replace(find, replace)
df.to_csv("csv_for_translate.csv", index = False)
self.ids.pb.value=0
self.parent.current='addgesture'

class GesturesListScreen(Screen):
customgestures=StringProperty("")
defaultgestures=StringProperty("")
customgestures=pd.read_csv('customgestureslist.csv', skiprows=1)
defaultgestures=pd.read_csv('defaultgestureslist.csv', skiprows=1)
customgestures=customgestures.to_string(index=False)
defaultgestures=defaultgestures.to_string(index=False)

def on_enter(self):
customgestures=StringProperty("")
defaultgestures=StringProperty("")
gesturecount=StringProperty("")
customcount=StringProperty("")
customgestures=pd.read_csv('customgestureslist.csv', skiprows=1)
customgestures = customgestures.drop_duplicates()
defaultgestures=pd.read_csv('defaultgestureslist.csv', skiprows=1)
count = str((len(defaultgestures)+1))
count2 = str((len(customgestures)+1))
customgestures=customgestures.to_string(index=False)
defaultgestures=defaultgestures.to_string(index=False)
#gesturecount = gesturecount.to_string(index=False)
# customgestures=str(customgestures)
# defaultgestures=str(defaultgestures)
df = pd.read_csv('csv_for_translate.csv', index_col = 'LABEL')
self.df=df
self.ids['defaultgestureskv'].text=str(defaultgestures)
self.ids['customgestureskv'].text=str(customgestures)
self.ids['gesturescount'].text=str(count)
self.ids['gesturescount2'].text=str(count2)

class DeleteGestureScreen(Screen):
customgestures=StringProperty("")
defaultgestures=StringProperty("")
customgestures=pd.read_csv('customgestureslist.csv')
defaultgestures=pd.read_csv('defaultgestureslist.csv')
df = pd.read_csv('csv_for_translate.csv', index_col = 'LABEL')
def on_enter(self):
customgestures=StringProperty("")
defaultgestures=StringProperty("")
 152

customgestures=pd.read_csv('customgestureslist.csv')
self.customgestures=customgestures
defaultgestures=pd.read_csv('defaultgestureslist.csv')
self.defaultgestures=defaultgestures
df = pd.read_csv('csv_for_translate.csv', index_col = 'LABEL')
self.df=df
def quit_screen(self):
self.ids.deletefail.opacity = 0
self.ids.indexabsent.opacity = 0
self.ids.deletesuccess.opacity = 0
def deletegesture(self):
deleteindex=StringProperty("")
deleteindex=self.ids.deleteindex.text
deleteindex=deleteindex.upper()
if (str(deleteindex) in self.defaultgestures.values):
self.ids.deletefail.opacity = 1
self.ids.indexabsent.opacity = 0
self.ids.deletesuccess.opacity = 0
elif (str(deleteindex) in self.customgestures.values):
if (str(deleteindex) in self.df.index.values):
self.ids.deletefail.opacity = 0
self.ids.indexabsent.opacity = 0
self.ids.deletesuccess.opacity = 1
self.df.drop(deleteindex, inplace = True) #delete rows by index
self.df.to_csv("csv_for_translate.csv", index = True)
customgestures=pd.read_csv('customgestureslist.csv')
custom_cols= customgestures['CUSTOM'].tolist()
f = open("customgestureslist.csv", "w") #empties the csv file
f.truncate()
f.close()
custom_cols.remove(str(deleteindex))
print custom_cols
with open('customgestureslist.csv', "wb") as cusfile:
writer = csv.writer(cusfile)
writer.writerow(['CUSTOM'])
for custom_col in custom_cols:
writer.writerow([custom_col])
else:
self.ids.deletefail.opacity = 0
self.ids.indexabsent.opacity = 1
self.ids.deletesuccess.opacity = 0
else:
self.ids.deletefail.opacity = 0
self.ids.indexabsent.opacity = 1
self.ids.deletesuccess.opacity = 0

class GestureGameScreen(Screen):
scorevar=StringProperty()
scorevar=0
scorevar=str(scorevar)
imageaddress=StringProperty("")
def on_enter(self):
imp.reload(sys.modules['modeling'])
ports = list(serial.tools.list_ports.comports())
self.scorenum=0
self.questioncount=1

# imageaddress=StringProperty("")
# imageaddress=""
# self.imageaddress=imageaddress
 153

self.ids['scoreboard'].text=str(self.scorenum)+"/10"
self.ids['questioncount'].text=str(self.questioncount)+"/10"
imp.reload(sys.modules['modeling'])
ports = list(serial.tools.list_ports.comports())
for p in ports:
comvar = str(p)
print p
if ('USB-SERIAL CH340') in comvar:
comportvar=comvar.split(' ', 1)[0]
break
print comportvar
# comportvar=str("'"+comportvar+"'")
self.comportvar=comportvar
c = threading.Condition()
self.c=c
global flag3
global num3
global lock3
lock3=1
num3=1
flag3= 1
global currentchoice
global newchoice
newchoice=0
self.thread1=threading.Thread(target=self.serialwriter)
self.thread1.start()
self.thread2=threading.Thread(target=self.serialvariable)
self.thread2.start()
def serialvariable(self):
global flag3
global randomchoice
while True:
self.c.acquire()
if (flag3 == 1):
print "random variable is contained"
#customgestures=pd.read_csv('customgestureslist.csv')
print "randomizer is now choosing"
self.ids.userdidthis.opacity = 0
self.ids.correct_image.opacity=0
self.ids.correctbg.opacity=0
self.ids.wrongbg.opacity=0
self.ids.questionbg.opacity=1
defaultgestures=pd.read_csv('defaultgestureslist.csv')
default_cols=defaultgestures['DEFAULT'].tolist()
#custom_cols.extend(default_cols)
global currentchoice
global randomchoice
randomflag=0
currentchoice= random.choice(default_cols)
self.currentchoice=currentchoice
self.ids['questiongesture'].text=str(self.currentchoice)
print self.currentchoice
randomchoice=self.currentchoice
flag3 = 0
self.c.notify_all()
time.sleep(2)
self.c.wait()
else:
self.c.wait()
self.c.release()
 154

def checker(self,*args):
print "checker is now comparing currentchoice and predvar"
global flag3
global newchoice
global lock3
global num3
print self.predvar
if flag3 == 0:
flag3=1
print "predvar"
print self.predvar
print "current choice"
print self.currentchoice
if (self.predvar == self.currentchoice):
self.scorevar=StringProperty()
print "match"
self.scorenum+=1
self.scorevar=str(self.scorenum)
self.ids['scoreboard'].text=str(self.scorevar)+"/10"
global finalscore
finalscore = self.scorevar
self.questioncount+=1
self.ids.correct_image.opacity=1
self.ids.correctbg.opacity=1
self.ids.wrongbg.opacity=0
self.ids.questionbg.opacity=0
self.ids.userdidthis.opacity = 1
self.ids.userdidthis.color = 0, 1, 0, 1
self.ids['userdidthis'].text=str(self.predvar)
self.ids['questioncount'].text=str(self.questioncount)+"/10"
engine = pyttsx.init()
engine.say(self.predvar)
engine.runAndWait()
flag3=1
newchoice=1
else:
finalscore = self.scorevar
self.questioncount+=1
self.ids.correct_image.opacity=1
self.ids.correctbg.opacity=0
self.ids.wrongbg.opacity=1
self.ids.questionbg.opacity=0
self.ids.userdidthis.opacity = 1
self.ids.userdidthis.color = 1, 0, 0, 1
self.ids['userdidthis'].text=str(self.predvar)
self.ids['questioncount'].text=str(self.questioncount)+"/10"
flag3=1
newchoice=1
print "serial received"
if (self.questioncount==11):
print "done"
lock3=0
num3=0
self.parent.current='gamescore'
def serial_stop(self):
#Clock.schedule_once(self.randomizer, 1)
global num3
global lock3
global flag3
 155

lock3=0
flag3=0
num3=0

class AboutMenuScreen(Screen):

def display_dev(self):
self.ids.app_layout.opacity = 0
self.ids.dev_layout.opacity = 1
self.ids.app_header.color = (0, 0.0, 0.0, 1.0)
def display_app(self):
self.ids.app_layout.opacity = 1
self.ids.dev_layout.opacity = 0
self.ids.dev_header.color = (0, 0.0, 0.0, 1.0)
def on_enter(self):
self.ids.app_layout.opacity = 0
self.ids.dev_layout.opacity = 0
def on_leave(self):
self.ids.app_layout.opacity = 0
self.ids.dev_layout.opacity = 0

class GameScoreScreen(Screen):
def on_enter(self):
global finalscore
lol = finalscore
print finalscore
self.ids['finalscore'].text=str(lol)+"/10"

sm = ScreenManager(transition=SwapTransition())
sm.add_widget(TriviaScreen(name='trivia'))
sm.add_widget(MenuScreen(name='menu'))
sm.add_widget(LearnScreen(name='learncategories'))
sm.add_widget(LearnWordScreen(name='learnwords'))
sm.add_widget(TranslateScreen(name='translate'))
sm.add_widget(AlphaPop1Screen(name='alphapop1'))
sm.add_widget(AlphaPopEndScreen(name='alphapopend'))
sm.add_widget(WordPop1Screen(name='wordpop1'))
sm.add_widget(WordPopEndScreen(name='wordpopend'))
sm.add_widget(AddGestureScreen(name='addgesture'))
sm.add_widget(CountdownScreen(name='countdown'))
sm.add_widget(PreGestureScreen(name='pregesture'))
sm.add_widget(PerformGestureScreen(name='performgesture'))
sm.add_widget(GesturesListScreen(name='gestureslist'))
sm.add_widget(DeleteGestureScreen(name='deletegesture'))
sm.add_widget(LearnLetterScreen(name='letterlearn'))
sm.add_widget(PreGameScreen(name='pregame'))
sm.add_widget(GestureGameScreen(name='gesturegame'))
sm.add_widget(AboutMenuScreen(name='aboutmenu'))
sm.add_widget(GameScoreScreen(name='gamescore'))

class MainApp(App):
def build(self):
self.title = 'FSL Translator'
return sm

if __name__ == '__main__':
MainApp().run()
 156

Python Data Modeling

import numpy as np
import pandas as pd
from sklearn.metrics import classification_report
from scipy import stats
from sklearn import svm
from sklearn import tree
from sklearn.ensemble import
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix

df= pd.read_csv("final.csv", delimiter=',')

from sklearn.model_selection import train_test_split

train, test = train_test_split(df, test_size = 0.5)

train_features =
train[['F1','F2','F3','F4','F5','X','Y','Z','C1','C2','lF1','lF2','lF3','lF4','lF5','lX','lY','lZ','lC1','lC2']]
train_label = train['LABEL']
test_features =
test[['F1','F2','F3','F4','F5','X','Y','Z','C1','C2','lF1','lF2','lF3','lF4','lF5','lX','lY','lZ','lC1','lC2']]
test_label = test['LABEL']

## SVM
model = svm.SVC(kernel='linear', gamma=0.1, C=1)
model.fit(train_features.values, train_label.values)
model.score(train_features, train_label)
predicted_svm = model.predict(test_features)