From 4266966e763a0ba62208167e274a32735738c13c Mon Sep 17 00:00:00 2001 From: Eduardo Robles Elvira Date: Fri, 13 May 2016 10:27:36 +0200 Subject: [PATCH 01/68] adding some extra options for answer formatting --- app/models/Models.scala | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/app/models/Models.scala b/app/models/Models.scala index 6b52dc72..ff986bb2 100644 --- a/app/models/Models.scala +++ b/app/models/Models.scala @@ -268,7 +268,23 @@ case class Question(description: String, layout: String, max: Int, min: Int, num } /** defines question extra data in an election */ -case class QuestionExtra(group: Option[String], next_button: Option[String], shuffled_categories: Option[String], shuffling_policy: Option[String], restrict_choices_by_tag__name: Option[String], restrict_choices_by_tag__max: Option[String], restrict_choices_by_tag__max_error_msg: Option[String], accordion_folding_policy: Option[String], restrict_choices_by_no_tag__max: Option[String], force_allow_blank_vote: Option[String], recommended_preset__tag: Option[String], recommended_preset__title: Option[String], recommended_preset__accept_text: Option[String], recommended_preset__deny_text: Option[String]) { +case class QuestionExtra( + group: Option[String], + next_button: Option[String], + shuffled_categories: Option[String], + shuffling_policy: Option[String], + restrict_choices_by_tag__name: Option[String], + restrict_choices_by_tag__max: Option[String], + restrict_choices_by_tag__max_error_msg: Option[String], + accordion_folding_policy: Option[String], + restrict_choices_by_no_tag__max: Option[String], + force_allow_blank_vote: Option[String], + recommended_preset__tag: Option[String], + recommended_preset__title: Option[String], + recommended_preset__accept_text: Option[String], + recommended_preset__deny_text: Option[String], + answer_columns_size: Option[String]) +{ def validate() = { assert(!group.isDefined || group.get.length <= SHORT_STRING, "group too long") @@ -288,6 +304,10 @@ case class QuestionExtra(group: Option[String], next_button: Option[String], shu assert(!recommended_preset__title.isDefined || recommended_preset__title.get.length <= LONG_STRING, "recommended_preset__title too long") assert(!recommended_preset__accept_text.isDefined || recommended_preset__accept_text.get.length <= LONG_STRING, "recommended_preset__accept_text too long") assert(!recommended_preset__deny_text.isDefined || recommended_preset__deny_text.get.length <= LONG_STRING, "recommended_preset__deny_text too long") + + assert(!answer_columns_size.isDefined || [12,6,4,3].contains(answer_columns_size.get.toInt), "invalid answer_columns_size, can only be a string with 12,6,4,3") + + assert(!group_answer_pairs.isDefined || group_answer_pairs.get.length <= SHORT_STRING, "group_answer_pairs too long") } } From ab0834d34b2165e8dcc3a28a36a9b3c441705563 Mon Sep 17 00:00:00 2001 From: Eduardo Robles Elvira Date: Sat, 14 May 2016 08:00:25 +0200 Subject: [PATCH 02/68] fixing list --- app/models/Models.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/Models.scala b/app/models/Models.scala index ff986bb2..9dfd09fc 100644 --- a/app/models/Models.scala +++ b/app/models/Models.scala @@ -305,7 +305,7 @@ case class QuestionExtra( assert(!recommended_preset__accept_text.isDefined || recommended_preset__accept_text.get.length <= LONG_STRING, "recommended_preset__accept_text too long") assert(!recommended_preset__deny_text.isDefined || recommended_preset__deny_text.get.length <= LONG_STRING, "recommended_preset__deny_text too long") - assert(!answer_columns_size.isDefined || [12,6,4,3].contains(answer_columns_size.get.toInt), "invalid answer_columns_size, can only be a string with 12,6,4,3") + assert(!answer_columns_size.isDefined || List(12,6,4,3).contains(answer_columns_size.get.toInt), "invalid answer_columns_size, can only be a string with 12,6,4,3") assert(!group_answer_pairs.isDefined || group_answer_pairs.get.length <= SHORT_STRING, "group_answer_pairs too long") } From 4822f86a7bfb47254606b8b1a4fe536e159e1faa Mon Sep 17 00:00:00 2001 From: Eduardo Robles Elvira Date: Sat, 14 May 2016 08:17:20 +0200 Subject: [PATCH 03/68] adding group_answer_pairs in the function call --- app/models/Models.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/Models.scala b/app/models/Models.scala index 9dfd09fc..01c68f6d 100644 --- a/app/models/Models.scala +++ b/app/models/Models.scala @@ -283,7 +283,8 @@ case class QuestionExtra( recommended_preset__title: Option[String], recommended_preset__accept_text: Option[String], recommended_preset__deny_text: Option[String], - answer_columns_size: Option[String]) + answer_columns_size: Option[String], + group_answer_pairs: Option[String]) { def validate() = { From 914b5d3ab973198b90b03710deb9be9627f6060f Mon Sep 17 00:00:00 2001 From: Findeton Date: Fri, 16 Dec 2016 11:19:31 +0100 Subject: [PATCH 04/68] set start/stop dates --- app/models/Models.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/models/Models.scala b/app/models/Models.scala index 67bebf76..e1d38cd3 100644 --- a/app/models/Models.scala +++ b/app/models/Models.scala @@ -215,7 +215,11 @@ object Elections { } def updateState(id: Long, state: String)(implicit s: Session) = { - elections.filter(_.id === id).map(e => e.state).update(state) + state match { + case STARTED => elections.filter(_.id === id).map(e => (e.state, e.startDate)).update(state, new Timestamp(new Date().getTime)) + case STOPPED => elections.filter(_.id === id).map(e => (e.state, e.endDate)).update(state, new Timestamp(new Date().getTime)) + case _ => elections.filter(_.id === id).map(e => e.state).update(state) + } } def updateResults(id: Long, results: String)(implicit s: Session) = { From 4a48f5780a881fa1e5596858fb11120f45553cb6 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 20 Dec 2016 15:20:34 +0100 Subject: [PATCH 05/68] allow publishing new results --- app/controllers/ElectionsApi.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/ElectionsApi.scala b/app/controllers/ElectionsApi.scala index cf61a38c..08503537 100644 --- a/app/controllers/ElectionsApi.scala +++ b/app/controllers/ElectionsApi.scala @@ -297,7 +297,9 @@ object ElectionsApi val future = getElection(id).flatMap { e => - if(e.state == Elections.TALLY_OK || e.state == Elections.RESULTS_OK) + if( Elections.TALLY_OK == e.state || + Elections.RESULTS_OK == e.state || + Elections.RESULTS_PUB == e.state ) { var electionConfigStr = Json.parse(e.configuration).as[JsObject] if (!electionConfigStr.as[JsObject].keys.contains("virtualSubelections")) From ec005c9ee8e26b2d1ae7867f1f3a8ad305280ce9 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 20 Dec 2016 16:27:41 +0100 Subject: [PATCH 06/68] allow calculate results after publishing --- app/controllers/ElectionsApi.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/ElectionsApi.scala b/app/controllers/ElectionsApi.scala index 08503537..ee5d358d 100644 --- a/app/controllers/ElectionsApi.scala +++ b/app/controllers/ElectionsApi.scala @@ -255,8 +255,8 @@ object ElectionsApi } case _ => if( - (e.state == Elections.TALLY_OK || e.state == Elections.RESULTS_OK) || - (e.virtual && e.state != Elections.RESULTS_PUB) + (Elections.TALLY_OK == e.state || Elections.RESULTS_OK == e.state || Elections.RESULTS_PUB == e.state) || + e.virtual ) { calcResults(id, config, validated.virtualSubelections.get).flatMap( r => updateResults(e, r) ) } From 98035004498372541b82a4796559a4cb5fc4b437 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 20 Dec 2016 17:51:44 +0100 Subject: [PATCH 07/68] don't allow calculate results after publishing them --- app/controllers/ElectionsApi.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/controllers/ElectionsApi.scala b/app/controllers/ElectionsApi.scala index ee5d358d..0fcecb0d 100644 --- a/app/controllers/ElectionsApi.scala +++ b/app/controllers/ElectionsApi.scala @@ -255,8 +255,8 @@ object ElectionsApi } case _ => if( - (Elections.TALLY_OK == e.state || Elections.RESULTS_OK == e.state || Elections.RESULTS_PUB == e.state) || - e.virtual + (Elections.TALLY_OK == e.state || Elections.RESULTS_OK == e.state) || + (e.virtual && e.state != Elections.RESULTS_PUB) ) { calcResults(id, config, validated.virtualSubelections.get).flatMap( r => updateResults(e, r) ) } @@ -298,8 +298,7 @@ object ElectionsApi { e => if( Elections.TALLY_OK == e.state || - Elections.RESULTS_OK == e.state || - Elections.RESULTS_PUB == e.state ) + Elections.RESULTS_OK == e.state) { var electionConfigStr = Json.parse(e.configuration).as[JsObject] if (!electionConfigStr.as[JsObject].keys.contains("virtualSubelections")) From 8e08df5bedf4f3f15d92dc690bde12600a0234db Mon Sep 17 00:00:00 2001 From: Eduardo Robles Elvira Date: Fri, 6 Jan 2017 10:43:39 +0100 Subject: [PATCH 08/68] adding more granular permissions for election creation, to be able to enable users to create test elections but not real elections --- app/controllers/ElectionsApi.scala | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/app/controllers/ElectionsApi.scala b/app/controllers/ElectionsApi.scala index cf61a38c..c653e383 100644 --- a/app/controllers/ElectionsApi.scala +++ b/app/controllers/ElectionsApi.scala @@ -81,17 +81,17 @@ object ElectionsApi val authorities = getAuthorityData /** inserts election into the db in the registered state */ - def register(id: Long) = HAction("", "AuthEvent", id, "edit").async(BodyParsers.parse.json) { request => + def register(id: Long) = HAction("", "AuthEvent", id, "edit|register").async(BodyParsers.parse.json) { request => registerElection(request, id) } /** updates an election's config */ - def update(id: Long) = HAction("", "AuthEvent", id, "edit").async(BodyParsers.parse.json) { request => + def update(id: Long) = HAction("", "AuthEvent", id, "edit|update").async(BodyParsers.parse.json) { request => updateElection(id, request) } /** updates an election's social share buttons config */ - def updateShare(id: Long) = HAction("", "AuthEvent", id, "edit").async(BodyParsers.parse.json) { request => + def updateShare(id: Long) = HAction("", "AuthEvent", id, "edit|update-share").async(BodyParsers.parse.json) { request => updateShareElection(id, request) } @@ -107,7 +107,7 @@ object ElectionsApi } /** Creates an election in eo */ - def create(id: Long) = HAction("", "AuthEvent", id, "edit").async { request => + def create(id: Long) = HAction("", "AuthEvent", id, "edit|create").async { request => getElection(id).flatMap(createElection).recover { @@ -487,6 +487,13 @@ object ElectionsApi { try { val validated = config.validate(authorities, id) + if (validated.real && !HMACAuthAction( + "", "AuthEvent", id, "edit|register-real" + ).validate(request)) + { + Logger.warn(s"Invalid config json, user has not permissions to register real elections") + BadRequest(error(s"Invalid config json")) + } DB.withSession { implicit session => @@ -599,6 +606,13 @@ object ElectionsApi config => { val validated = config.validate(authorities, id) + if (validated.real && !HMACAuthAction( + "", "AuthEvent", id, "edit|register-real" + ).validate(request)) + { + Logger.warn(s"Invalid config json, user has not permissions to register real elections") + BadRequest(error(s"Invalid config json")) + } val result = DAL.elections.updateConfig(id, validated.asString, validated.start_date, validated.end_date) Ok(response(result)) From 28f67da4b2c0fdfac6c859e1b309362d1e692a09 Mon Sep 17 00:00:00 2001 From: Eduardo Robles Elvira Date: Fri, 6 Jan 2017 11:11:28 +0100 Subject: [PATCH 09/68] trying to make it compile --- app/controllers/ElectionsApi.scala | 44 +++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/app/controllers/ElectionsApi.scala b/app/controllers/ElectionsApi.scala index c653e383..4e035674 100644 --- a/app/controllers/ElectionsApi.scala +++ b/app/controllers/ElectionsApi.scala @@ -487,12 +487,24 @@ object ElectionsApi { try { val validated = config.validate(authorities, id) - if (validated.real && !HMACAuthAction( - "", "AuthEvent", id, "edit|register-real" - ).validate(request)) + if (validated.real) { - Logger.warn(s"Invalid config json, user has not permissions to register real elections") - BadRequest(error(s"Invalid config json")) + request + .headers + .get("Authorization") + .map( + HMACAuthAction("", "AuthEvent", id, "edit|register-real") + .validate(request) + ) match + { + case Some(true) => None + case _ => BadRequest( + error( + s"Invalid config json, user has not permissions to " + + s"register real elections" + ) + ) + } } DB.withSession { @@ -606,12 +618,24 @@ object ElectionsApi config => { val validated = config.validate(authorities, id) - if (validated.real && !HMACAuthAction( - "", "AuthEvent", id, "edit|register-real" - ).validate(request)) + if (validated.real) { - Logger.warn(s"Invalid config json, user has not permissions to register real elections") - BadRequest(error(s"Invalid config json")) + request + .headers + .get("Authorization") + .map( + HMACAuthAction("", "AuthEvent", id, "edit|register-real") + .validate(request) + ) match + { + case Some(true) => None + case _ => BadRequest( + error( + s"Invalid config json, user has not permissions to " + + s"register real elections" + ) + ) + } } val result = DAL.elections.updateConfig(id, validated.asString, validated.start_date, validated.end_date) From dd73cfa1142c244191b9f14166100a51683a4a33 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 17 Jan 2017 17:11:22 +0100 Subject: [PATCH 10/68] improve BallotBox.vote code --- app/controllers/BallotboxApi.scala | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/controllers/BallotboxApi.scala b/app/controllers/BallotboxApi.scala index 2fb3a758..e73c4512 100644 --- a/app/controllers/BallotboxApi.scala +++ b/app/controllers/BallotboxApi.scala @@ -87,9 +87,8 @@ object BallotboxApi extends Controller with Response { val validated = vote.validate(pks, true, electionId, voterId) val result = DAL.votes.insertWithSession(validated) - val now = new java.util.Date().getTime / 1000 - val message = "$voterId:AuthEvent:$electionId:RegisterSuccessfulLogin:$now" - val timestamp: Long = System.currentTimeMillis / 1000 + val now: Long = System.currentTimeMillis / 1000 + val message = s"$voterId:AuthEvent:$electionId:RegisterSuccessfulLogin:$now" voteCallbackUrl.map { url => postVoteCallback( url @@ -97,9 +96,6 @@ object BallotboxApi extends Controller with Response { .replace("${uid}", voterId) , message - .replace("$voterId", voterId) - .replace("$electionId", electionId+"") - .replace("$now", timestamp+"") ) } Ok(response(result)) From ba5f447b9a051e39df6e180b970356d5edca1698 Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 2 Feb 2017 17:32:36 +0100 Subject: [PATCH 11/68] add gen votes --- admin/admin.py | 90 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/admin/admin.py b/admin/admin.py index dadb6226..c98f41e5 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -31,6 +31,7 @@ import hashlib import codecs import traceback +import tempfile import os.path import os @@ -636,13 +637,96 @@ def change_social(cfg, args): print("invalid share-config file %s" % args.share_config) return 400 +def gen_votes(cfg, args): + def _open(path, mode): + return codecs.open(path, encoding='utf-8', mode=mode) + + def _read_file(path): + _check_file(path) + with _open(path, mode='r') as f: + return f.read() + + def _check_file(path): + if not os.access(path, os.R_OK): + raise Exception("Error: can't read %s" % path) + if not os.path.isfile(path): + raise Exception("Error: not a file %s" % path) + + def _write_file(path, data): + with _open(path, mode='w') as f: + return f.write(data) + + def gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count): + base_plaintexts = [ d.strip() for d in _read_file(base_plaintexts_path).splitlines() ] + new_plaintext_path = os.path.join(temp_path, 'plaintext') + new_plaintext = "" + counter = 0 + mod_base = len(base_plaintexts) + while counter < vote_count: + new_plaintext += "%s\n" % (base_plaintexts[counter % mod_base]) + counter += 1 + _write_file(new_plaintext_path, new_plaintext) + return new_plaintext_path + + def gen_all_khmacs(vote_count, election_id): + import hmac + secret = shared_secret + alphabet = '0123456789abcdef' + timestamp = 1000 * int(time.time()) + counter = 0 + khmac_list = [] + while counter < vote_count: + voterid = ''.join(alphabet[c % len(alphabet)] for c in os.urandom(length)) + message = '%s:AuthEvent:%i:vote:%s' % (voterid, election_id, timestamp) + _hmac = hmac.new(str.encode(secret), str.encode(message), hashlib.sha256).hexdigest() + khmac = 'khmac:///sha-256;%s/%s' % (_hmac, message) + khmac_list.append((voterid, khmac)) + counter += 1 + + return khmac_list + + def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): + cyphertexts_list = _read_file(ciphertexts_path).splitlines() + host,port = get_local_hostport() + for index, ballot in enumerate(cyphertexts_list): + if index >= vote_count: + break + voterid, khmac = khmac_list[index] + headers = { + 'content-type': 'application/json', + 'Authorization': khmac + } + url = 'http://%s:%d/elections/api/election/%i/voter/%s' % (host, port, election_id, voterid) + r = requests.post(url, data=ballot, headers=headers) + if r.status_code != 200: + raise Exception("Error voting: HTTP POST to %s with khmac %s returned code %i and error %s" % (url, khmac, r.status_code, r.text[:200])) + + if args.vote_count <= 0: + raise Exception("vote count must be > 0") + + _check_file(config['plaintexts']) + with tempfile.TemporaryDirectory() as temp_path: + # a list of base plaintexts to generate + base_plaintexts_path = cfg["plaintexts"] + election_id = cfg['election_id'] + vote_count = args.vote_count + ciphertexts_path = os.path.join(temp_path, 'ciphertexts') + cfg["plaintexts"] = gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count) + khmac_list = gen_all_khmacs(vote_count, election_id) + cfg['encrypt-count'] = vote_count + cfg['ciphertexts'] = ciphertexts_path + dump_pks(cfg, args) + encrypt(cfg, args) + khmac_list = gen_all_khmacs(vote_count, election_id) + send_all_ballots((vote_count, ciphertexts_path, khmac_list, election_id) + def get_hmac(cfg, userId, objType, objId, perm): import hmac secret = shared_secret - now = 1000*long(time.time()) + now = 1000*int(time.time()) message = "%s:%s:%d:%s:%d" % (userId, objType, objId, perm, now) - _hmac = hmac.new(str(secret), str(message), hashlib.sha256).hexdigest() + _hmac = hmac.new(str.encode(secret), str.encode(message), hashlib.sha256).hexdigest() ret = 'khmac:///sha-256;%s/%s' % (_hmac, message) return ret @@ -677,12 +761,14 @@ def main(argv): dump_votes : dumps votes for an election (private datastore) change_social : changes the social netoworks share buttons configuration authapi_ensure_acls --acls-path : ensure that the acls inside acl_path exist. +gen_votes : generate votes ''') parser.add_argument('--ciphertexts', help='file to write ciphertetxs (used in dump, load and encrypt)') parser.add_argument('--acls-path', help='''the file has one line per acl with format: '(email:email@example.com|tlf:+34666777888),permission_name,object_type,object_id,user_election_id' ''') parser.add_argument('--plaintexts', help='json file to read votes from when encrypting', default = 'votes.json') parser.add_argument('--filter-config', help='file with filter configuration', default = None) parser.add_argument('--encrypt-count', help='number of votes to encrypt (generates duplicates if more than in json file)', type=int, default = 0) + parser.add_argument('--vote-count', help='number of votes to generate', type=int, default = 0) parser.add_argument('--results-config', help='config file for agora-results') parser.add_argument('--voter-ids', help='json file with list of valid voter ids to tally (used with tally_voter_ids)') parser.add_argument('--ips-log', help='') From 85c0692c60a15aa0f8c74b639376cee40a53c95b Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 2 Feb 2017 17:42:21 +0100 Subject: [PATCH 12/68] add prints --- admin/admin.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/admin/admin.py b/admin/admin.py index c98f41e5..f70c3d5a 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -707,18 +707,25 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): _check_file(config['plaintexts']) with tempfile.TemporaryDirectory() as temp_path: # a list of base plaintexts to generate + print("created temporary folder at %s" % temp_path) base_plaintexts_path = cfg["plaintexts"] election_id = cfg['election_id'] vote_count = args.vote_count ciphertexts_path = os.path.join(temp_path, 'ciphertexts') cfg["plaintexts"] = gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count) + print("created plaintexts") khmac_list = gen_all_khmacs(vote_count, election_id) + print("created khmac") cfg['encrypt-count'] = vote_count cfg['ciphertexts'] = ciphertexts_path dump_pks(cfg, args) + print("pks dumped") encrypt(cfg, args) + print("ballotes encrypted") khmac_list = gen_all_khmacs(vote_count, election_id) + print("khmacs generated") send_all_ballots((vote_count, ciphertexts_path, khmac_list, election_id) + print("ballots sent") def get_hmac(cfg, userId, objType, objId, perm): import hmac From 864bd81d74150c7a5f3504e1804bf849a84699d3 Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 2 Feb 2017 17:43:11 +0100 Subject: [PATCH 13/68] fix syntax --- admin/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/admin.py b/admin/admin.py index f70c3d5a..2404ce8e 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -724,7 +724,7 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): print("ballotes encrypted") khmac_list = gen_all_khmacs(vote_count, election_id) print("khmacs generated") - send_all_ballots((vote_count, ciphertexts_path, khmac_list, election_id) + send_all_ballots((vote_count, ciphertexts_path, khmac_list, election_id)) print("ballots sent") def get_hmac(cfg, userId, objType, objId, perm): From f557a44e81733455366c60efa4a16a05de9ef9a9 Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 2 Feb 2017 19:08:13 +0100 Subject: [PATCH 14/68] fixes to gen votes --- admin/admin.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/admin/admin.py b/admin/admin.py index 2404ce8e..680646eb 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -608,7 +608,7 @@ def encrypt(cfg, args): output, error = subprocess.Popen(["bash", "encrypt.sh", pkPath, votesPath, str(votesCount)], stdout = subprocess.PIPE).communicate() print("Received encrypt.sh output (" + str(len(output)) + " chars)") - parsed = json.loads(output) + parsed = json.loads(output.decode('utf-8')) print("Writing file to " + ctextsPath) with codecs.open(ctextsPath, encoding='utf-8', mode='w+') as votes_file: @@ -659,11 +659,14 @@ def _write_file(path, data): def gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count): base_plaintexts = [ d.strip() for d in _read_file(base_plaintexts_path).splitlines() ] new_plaintext_path = os.path.join(temp_path, 'plaintext') - new_plaintext = "" + new_plaintext = "[\n" counter = 0 mod_base = len(base_plaintexts) while counter < vote_count: - new_plaintext += "%s\n" % (base_plaintexts[counter % mod_base]) + if counter + 1 == vote_count: + new_plaintext += "%s]" % (base_plaintexts[counter % mod_base]) + else: + new_plaintext += "%s,\n" % (base_plaintexts[counter % mod_base]) counter += 1 _write_file(new_plaintext_path, new_plaintext) return new_plaintext_path @@ -675,11 +678,11 @@ def gen_all_khmacs(vote_count, election_id): timestamp = 1000 * int(time.time()) counter = 0 khmac_list = [] + voterid_len = 28 while counter < vote_count: - voterid = ''.join(alphabet[c % len(alphabet)] for c in os.urandom(length)) + voterid = ''.join(alphabet[c % len(alphabet)] for c in os.urandom(voterid_len)) message = '%s:AuthEvent:%i:vote:%s' % (voterid, election_id, timestamp) - _hmac = hmac.new(str.encode(secret), str.encode(message), hashlib.sha256).hexdigest() - khmac = 'khmac:///sha-256;%s/%s' % (_hmac, message) + khmac = get_hmac(cfg, voterid, "AuthEvent", election_id, 'vote') khmac_list.append((voterid, khmac)) counter += 1 @@ -688,7 +691,7 @@ def gen_all_khmacs(vote_count, election_id): def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): cyphertexts_list = _read_file(ciphertexts_path).splitlines() host,port = get_local_hostport() - for index, ballot in enumerate(cyphertexts_list): + for index, vote in enumerate(cyphertexts_list): if index >= vote_count: break voterid, khmac = khmac_list[index] @@ -696,7 +699,12 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): 'content-type': 'application/json', 'Authorization': khmac } - url = 'http://%s:%d/elections/api/election/%i/voter/%s' % (host, port, election_id, voterid) + vote_hash = hashlib.sha256(vote_string).hexdigest() + ballot = json.dumps({ + "vote": json.dumps(vote), + "vote_hash": vote_hash + }) + url = 'http://%s:%d/api/election/%i/voter/%s' % (host, port, election_id, voterid) r = requests.post(url, data=ballot, headers=headers) if r.status_code != 200: raise Exception("Error voting: HTTP POST to %s with khmac %s returned code %i and error %s" % (url, khmac, r.status_code, r.text[:200])) @@ -704,7 +712,7 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): if args.vote_count <= 0: raise Exception("vote count must be > 0") - _check_file(config['plaintexts']) + _check_file(cfg['plaintexts']) with tempfile.TemporaryDirectory() as temp_path: # a list of base plaintexts to generate print("created temporary folder at %s" % temp_path) @@ -724,7 +732,7 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): print("ballotes encrypted") khmac_list = gen_all_khmacs(vote_count, election_id) print("khmacs generated") - send_all_ballots((vote_count, ciphertexts_path, khmac_list, election_id)) + send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id) print("ballots sent") def get_hmac(cfg, userId, objType, objId, perm): @@ -762,6 +770,7 @@ def main(argv): dump_votes [election_id, [election_id], ...]: dump voter ids list_votes : list votes list_elections: list elections +cast_votes : cast votes from ciphertetxs dump_pks : dumps pks for an election (public datastore) encrypt : encrypts votes using scala (public key must be in datastore) encryptNode : encrypts votes using node (public key must be in datastore) From fa009c9988f5aa660dbf49e1c35cc503f746515b Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 2 Feb 2017 19:50:46 +0100 Subject: [PATCH 15/68] fixes --- admin/admin.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/admin/admin.py b/admin/admin.py index 680646eb..25f3a70c 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -287,7 +287,7 @@ def cast_votes(cfg, args): vote_string = json.dumps(vote) vote_hash = hashlib.sha256(vote_string).hexdigest() vote = { - "vote": json.dumps(vote), + "vote": vote_string, "vote_hash": vote_hash } @@ -673,7 +673,6 @@ def gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count): def gen_all_khmacs(vote_count, election_id): import hmac - secret = shared_secret alphabet = '0123456789abcdef' timestamp = 1000 * int(time.time()) counter = 0 @@ -689,9 +688,10 @@ def gen_all_khmacs(vote_count, election_id): return khmac_list def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): - cyphertexts_list = _read_file(ciphertexts_path).splitlines() + cyphertexts_json = json.loads(_read_file(ciphertexts_path)) host,port = get_local_hostport() - for index, vote in enumerate(cyphertexts_list): + for index, vote in enumerate(cyphertexts_json): + vote_string = json.dumps(vote) if index >= vote_count: break voterid, khmac = khmac_list[index] @@ -699,15 +699,15 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): 'content-type': 'application/json', 'Authorization': khmac } - vote_hash = hashlib.sha256(vote_string).hexdigest() - ballot = json.dumps({ - "vote": json.dumps(vote), + vote_hash = hashlib.sha256(str.encode(vote_string)).hexdigest() + json.dumps({ + "vote": vote_string, "vote_hash": vote_hash }) url = 'http://%s:%d/api/election/%i/voter/%s' % (host, port, election_id, voterid) r = requests.post(url, data=ballot, headers=headers) if r.status_code != 200: - raise Exception("Error voting: HTTP POST to %s with khmac %s returned code %i and error %s" % (url, khmac, r.status_code, r.text[:200])) + raise Exception("Error voting: HTTP POST to %s with khmac %s returned code %i, error %s, http data: %s" % (url, khmac, r.status_code, r.text[:200], ballot)) if args.vote_count <= 0: raise Exception("vote count must be > 0") From c389935997d8d79ed61b950f40647a994bbac9fd Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 2 Feb 2017 19:52:41 +0100 Subject: [PATCH 16/68] fix typo --- admin/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/admin.py b/admin/admin.py index 25f3a70c..dccc0843 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -700,7 +700,7 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): 'Authorization': khmac } vote_hash = hashlib.sha256(str.encode(vote_string)).hexdigest() - json.dumps({ + ballot = json.dumps({ "vote": vote_string, "vote_hash": vote_hash }) From ecf1313129d099ed32ff40d263c3313766bfc49c Mon Sep 17 00:00:00 2001 From: Eduardo Robles Elvira Date: Fri, 3 Feb 2017 10:01:06 +0100 Subject: [PATCH 17/68] trying to make it work with python2 --- admin/admin.py | 105 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 3 deletions(-) diff --git a/admin/admin.py b/admin/admin.py index dccc0843..dd4891d7 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with agora_elections. If not, see . +from __future__ import print_function import requests import json import time @@ -31,12 +32,18 @@ import hashlib import codecs import traceback -import tempfile +import string import os.path import os from prettytable import PrettyTable + +import warnings as _warnings +import os as _os + +from tempfile import mkdtemp + from sqlalchemy import create_engine, select, func, text from sqlalchemy import Table, Column, Integer, String, TIMESTAMP, MetaData, ForeignKey from sqlalchemy import distinct @@ -61,6 +68,92 @@ authapi_db_port = 5432 node = '/usr/local/bin/node' +class TemporaryDirectory(object): + """Create and return a temporary directory. This has the same + behavior as mkdtemp but can be used as a context manager. For + example: + + with TemporaryDirectory() as tmpdir: + ... + + Upon exiting the context, the directory and everything contained + in it are removed. + """ + + def __init__(self, suffix="", prefix="tmp", dir=None): + self._closed = False + self.name = None # Handle mkdtemp raising an exception + self.name = mkdtemp(suffix, prefix, dir) + + def __repr__(self): + return "<{} {!r}>".format(self.__class__.__name__, self.name) + + def __enter__(self): + return self.name + + def cleanup(self, _warn=False): + if self.name and not self._closed: + try: + self._rmtree(self.name) + except (TypeError, AttributeError) as ex: + # Issue #10188: Emit a warning on stderr + # if the directory could not be cleaned + # up due to missing globals + if "None" not in str(ex): + raise + print("ERROR: {!r} while cleaning up {!r}".format(ex, self,), + file=_sys.stderr) + return + self._closed = True + if _warn: + self._warn("Implicitly cleaning up {!r}".format(self), + ResourceWarning) + def __exit__(self, exc, value, tb): + self.cleanup() + + def __del__(self): + # Issue a ResourceWarning if implicit cleanup needed + self.cleanup(_warn=True) + + # XXX (ncoghlan): The following code attempts to make + # this class tolerant of the module nulling out process + # that happens during CPython interpreter shutdown + # Alas, it doesn't actually manage it. See issue #10188 + _listdir = staticmethod(_os.listdir) + _path_join = staticmethod(_os.path.join) + _isdir = staticmethod(_os.path.isdir) + _islink = staticmethod(_os.path.islink) + _remove = staticmethod(_os.remove) + _rmdir = staticmethod(_os.rmdir) + _warn = _warnings.warn + + def _rmtree(self, path): + # Essentially a stripped down version of shutil.rmtree. We can't + # use globals because they may be None'ed out at shutdown. + for name in self._listdir(path): + fullname = self._path_join(path, name) + try: + isdir = self._isdir(fullname) and not self._islink(fullname) + except OSError: + isdir = False + if isdir: + self._rmtree(fullname) + else: + try: + self._remove(fullname) + except OSError: + pass + try: + self._rmdir(path) + except OSError: + pass + + + + + + + def get_local_hostport(): return app_host, app_port @@ -671,6 +764,12 @@ def gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count): _write_file(new_plaintext_path, new_plaintext) return new_plaintext_path + def gen_rnd_str(length, choices): + return ''.join( + random.SystemRandom().choice(string.ascii_uppercase + string.digits) + for _ in range(length) + ) + def gen_all_khmacs(vote_count, election_id): import hmac alphabet = '0123456789abcdef' @@ -679,7 +778,7 @@ def gen_all_khmacs(vote_count, election_id): khmac_list = [] voterid_len = 28 while counter < vote_count: - voterid = ''.join(alphabet[c % len(alphabet)] for c in os.urandom(voterid_len)) + voterid = gen_rnd_str(voterid_len, alphabet) message = '%s:AuthEvent:%i:vote:%s' % (voterid, election_id, timestamp) khmac = get_hmac(cfg, voterid, "AuthEvent", election_id, 'vote') khmac_list.append((voterid, khmac)) @@ -713,7 +812,7 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): raise Exception("vote count must be > 0") _check_file(cfg['plaintexts']) - with tempfile.TemporaryDirectory() as temp_path: + with TemporaryDirectory() as temp_path: # a list of base plaintexts to generate print("created temporary folder at %s" % temp_path) base_plaintexts_path = cfg["plaintexts"] From f2a58dfd7b66caa822b99bf5724cdd97e3e46721 Mon Sep 17 00:00:00 2001 From: Findeton Date: Fri, 3 Feb 2017 10:57:58 +0100 Subject: [PATCH 18/68] removed duplicated khmac generation --- admin/admin.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/admin/admin.py b/admin/admin.py index dd4891d7..fd6b9c87 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -822,15 +822,13 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): cfg["plaintexts"] = gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count) print("created plaintexts") khmac_list = gen_all_khmacs(vote_count, election_id) - print("created khmac") + print("created khmacs") cfg['encrypt-count'] = vote_count cfg['ciphertexts'] = ciphertexts_path dump_pks(cfg, args) print("pks dumped") encrypt(cfg, args) - print("ballotes encrypted") - khmac_list = gen_all_khmacs(vote_count, election_id) - print("khmacs generated") + print("ballots encrypted") send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id) print("ballots sent") From a4ac1afca915f64c8c6ab8da7befc2cff11e38b5 Mon Sep 17 00:00:00 2001 From: Findeton Date: Fri, 3 Feb 2017 11:15:50 +0100 Subject: [PATCH 19/68] fix random --- admin/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/admin/admin.py b/admin/admin.py index fd6b9c87..473d56f5 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -37,7 +37,7 @@ import os.path import os from prettytable import PrettyTable - +import random import warnings as _warnings import os as _os @@ -766,7 +766,7 @@ def gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count): def gen_rnd_str(length, choices): return ''.join( - random.SystemRandom().choice(string.ascii_uppercase + string.digits) + random.SystemRandom().choice(choices) for _ in range(length) ) From 99d258dc7e0f734af640ccf3f7da7ff51019f7da Mon Sep 17 00:00:00 2001 From: Findeton Date: Fri, 3 Feb 2017 12:17:21 +0100 Subject: [PATCH 20/68] refactor to save/read ciphertexts --- admin/admin.py | 60 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/admin/admin.py b/admin/admin.py index 473d56f5..6d157c6c 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -38,6 +38,7 @@ import os from prettytable import PrettyTable import random +import shutil import warnings as _warnings import os as _os @@ -772,6 +773,7 @@ def gen_rnd_str(length, choices): def gen_all_khmacs(vote_count, election_id): import hmac + start_time = time.time() alphabet = '0123456789abcdef' timestamp = 1000 * int(time.time()) counter = 0 @@ -784,9 +786,15 @@ def gen_all_khmacs(vote_count, election_id): khmac_list.append((voterid, khmac)) counter += 1 + end_time = time.time() + delta_t = end_time - start_time + troughput = vote_count / float(delta_t) + print("created %i khmacs in %f secs. %f khmacs/sec" % (vote_count, delta_t, troughput)) + return khmac_list def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): + start_time = time.time() cyphertexts_json = json.loads(_read_file(ciphertexts_path)) host,port = get_local_hostport() for index, vote in enumerate(cyphertexts_json): @@ -807,28 +815,54 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): r = requests.post(url, data=ballot, headers=headers) if r.status_code != 200: raise Exception("Error voting: HTTP POST to %s with khmac %s returned code %i, error %s, http data: %s" % (url, khmac, r.status_code, r.text[:200], ballot)) + end_time = time.time() + delta_t = end_time - start_time + troughput = vote_count / float(delta_t) + print("sent %i votes in %f secs. %f votes/sec" % (vote_count, delta_t, troughput)) if args.vote_count <= 0: raise Exception("vote count must be > 0") - _check_file(cfg['plaintexts']) with TemporaryDirectory() as temp_path: - # a list of base plaintexts to generate - print("created temporary folder at %s" % temp_path) - base_plaintexts_path = cfg["plaintexts"] + print("created temporary folder at %s" % temp_path) + election_id = cfg['election_id'] vote_count = args.vote_count - ciphertexts_path = os.path.join(temp_path, 'ciphertexts') - cfg["plaintexts"] = gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count) - print("created plaintexts") - khmac_list = gen_all_khmacs(vote_count, election_id) - print("created khmacs") + + save_ciphertexts = False + read_ciphertexts = False + if cfg['ciphertexts'] is not None: + # if the file exists, then read it: we don't need to generate it + if os.access(cfg['ciphertexts'], os.R_OK) and os.path.isfile(cfg['ciphertexts']): + read_ciphertexts = True + ciphertexts_path = cfg['ciphertexts'] + # if it doesn't exist, generate the ciphertexts and save them there + else: + _check_file(cfg['plaintexts']) + save_ciphertexts = True + save_ciphertexts_path = cfg['ciphertexts'] + else: + _check_file(cfg['plaintexts']) + cfg['encrypt-count'] = vote_count cfg['ciphertexts'] = ciphertexts_path - dump_pks(cfg, args) - print("pks dumped") - encrypt(cfg, args) - print("ballots encrypted") + + if not read_ciphertexts: + # a list of base plaintexts to generate ballots + base_plaintexts_path = cfg["plaintexts"] + ciphertexts_path = os.path.join(temp_path, 'ciphertexts') + cfg["plaintexts"] = gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count) + print("created plaintexts") + dump_pks(cfg, args) + print("pks dumped") + encrypt(cfg, args) + print("ballots encrypted") + if save_ciphertexts: + shutil.copy2(ciphertexts_path, save_ciphertexts_path) + print("encrypted ballots saved to file %s" % ciphertexts_path) + + khmac_list = gen_all_khmacs(vote_count, election_id) + print("created khmacs") send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id) print("ballots sent") From 204b45a02884d506eddba3d71d72c44e73f92d12 Mon Sep 17 00:00:00 2001 From: Findeton Date: Fri, 3 Feb 2017 12:21:53 +0100 Subject: [PATCH 21/68] fixes --- admin/admin.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/admin/admin.py b/admin/admin.py index 6d157c6c..d80586f7 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -828,6 +828,7 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): election_id = cfg['election_id'] vote_count = args.vote_count + cfg['encrypt-count'] = vote_count save_ciphertexts = False read_ciphertexts = False @@ -844,19 +845,23 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): else: _check_file(cfg['plaintexts']) - cfg['encrypt-count'] = vote_count - cfg['ciphertexts'] = ciphertexts_path if not read_ciphertexts: # a list of base plaintexts to generate ballots base_plaintexts_path = cfg["plaintexts"] ciphertexts_path = os.path.join(temp_path, 'ciphertexts') + cfg['ciphertexts'] = ciphertexts_path cfg["plaintexts"] = gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count) print("created plaintexts") dump_pks(cfg, args) print("pks dumped") + start_time = time.time() encrypt(cfg, args) print("ballots encrypted") + end_time = time.time() + delta_t = end_time - start_time + troughput = vote_count / float(delta_t) + print("encrypted %i votes in %f secs. %f votes/sec" % (vote_count, delta_t, troughput)) if save_ciphertexts: shutil.copy2(ciphertexts_path, save_ciphertexts_path) print("encrypted ballots saved to file %s" % ciphertexts_path) From 1269cdc12a7a693d41afafd070280aa315e38099 Mon Sep 17 00:00:00 2001 From: Findeton Date: Fri, 3 Feb 2017 13:02:04 +0100 Subject: [PATCH 22/68] improve prints --- admin/admin.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/admin/admin.py b/admin/admin.py index d80586f7..697982ab 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -824,7 +824,7 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): raise Exception("vote count must be > 0") with TemporaryDirectory() as temp_path: - print("created temporary folder at %s" % temp_path) + print("%s created temporary folder at %s" % (str(datetime.now()), temp_path)) election_id = cfg['election_id'] vote_count = args.vote_count @@ -851,13 +851,16 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): base_plaintexts_path = cfg["plaintexts"] ciphertexts_path = os.path.join(temp_path, 'ciphertexts') cfg['ciphertexts'] = ciphertexts_path + print("%s start generation of plaintexts" % str(datetime.now())) cfg["plaintexts"] = gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count) - print("created plaintexts") + print("%s plaintexts created" % str(datetime.now())) + print("%s start dump_pks" % str(datetime.now())) dump_pks(cfg, args) - print("pks dumped") + print("%s pks dumped" % str(datetime.now())) + print("%s start ballot encryption" % str(datetime.now())) start_time = time.time() encrypt(cfg, args) - print("ballots encrypted") + print("%s ballots encrypted" % str(datetime.now())) end_time = time.time() delta_t = end_time - start_time troughput = vote_count / float(delta_t) @@ -866,10 +869,12 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): shutil.copy2(ciphertexts_path, save_ciphertexts_path) print("encrypted ballots saved to file %s" % ciphertexts_path) + print("%s start khmac generation" % str(datetime.now())) khmac_list = gen_all_khmacs(vote_count, election_id) - print("created khmacs") + print("%s khmacs generated" % str(datetime.now())) + print("%s start sending ballots" % str(datetime.now())) send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id) - print("ballots sent") + print("%s ballots sent" % str(datetime.now())) def get_hmac(cfg, userId, objType, objId, perm): import hmac From 42b160afd16da8c8238811a9a933834de28262f0 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 7 Feb 2017 16:10:54 +0100 Subject: [PATCH 23/68] separate steps for generating and sending votes --- admin/admin.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/admin/admin.py b/admin/admin.py index 697982ab..183dfc6f 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -868,13 +868,13 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): if save_ciphertexts: shutil.copy2(ciphertexts_path, save_ciphertexts_path) print("encrypted ballots saved to file %s" % ciphertexts_path) - - print("%s start khmac generation" % str(datetime.now())) - khmac_list = gen_all_khmacs(vote_count, election_id) - print("%s khmacs generated" % str(datetime.now())) - print("%s start sending ballots" % str(datetime.now())) - send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id) - print("%s ballots sent" % str(datetime.now())) + else: + print("%s start khmac generation" % str(datetime.now())) + khmac_list = gen_all_khmacs(vote_count, election_id) + print("%s khmacs generated" % str(datetime.now())) + print("%s start sending ballots" % str(datetime.now())) + send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id) + print("%s ballots sent" % str(datetime.now())) def get_hmac(cfg, userId, objType, objId, perm): import hmac From e34d2d99ad7fac8353bba87e9bc3d8bc1e6b7626 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 7 Feb 2017 19:13:43 +0100 Subject: [PATCH 24/68] support multiple questions for genvotes --- admin/admin.py | 6 +++-- app/commands/DemoVotes.scala | 5 ++-- app/utils/Crypto.scala | 50 +++++++++++++++++++++--------------- 3 files changed, 36 insertions(+), 25 deletions(-) diff --git a/admin/admin.py b/admin/admin.py index 183dfc6f..25f705de 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -757,10 +757,12 @@ def gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count): counter = 0 mod_base = len(base_plaintexts) while counter < vote_count: + base_ballot = base_plaintexts[counter % mod_base] + json_ballot = "[%s]" % base_ballot if counter + 1 == vote_count: - new_plaintext += "%s]" % (base_plaintexts[counter % mod_base]) + new_plaintext += "%s]" % json_ballot else: - new_plaintext += "%s,\n" % (base_plaintexts[counter % mod_base]) + new_plaintext += "%s,\n" % json_ballot counter += 1 _write_file(new_plaintext_path, new_plaintext) return new_plaintext_path diff --git a/app/commands/DemoVotes.scala b/app/commands/DemoVotes.scala index f0c45039..96e52d56 100644 --- a/app/commands/DemoVotes.scala +++ b/app/commands/DemoVotes.scala @@ -41,10 +41,9 @@ object DemoVotes { def main(args: Array[String]) : Unit = { val jsonPks = Json.parse(scala.io.Source.fromFile(args(0)).mkString) val pks = jsonPks.validate[Array[PublicKey]].get - val pk = pks(0) val jsonVotes = Json.parse(scala.io.Source.fromFile(args(1)).mkString) - val votes = jsonVotes.validate[Array[Long]].get + val votes = jsonVotes.validate[Array[Array[Long]]].get val toEncrypt = if(args.length == 3) { val extraSize = (args(2).toInt - votes.length).max(0) @@ -57,7 +56,7 @@ object DemoVotes { val histogram = toEncrypt.groupBy(l => l).map(t => (t._1, t._2.length)) System.err.println("DemoVotes: tally: " + histogram) - val jsonEncrypted = Json.toJson(toEncrypt.par.map(Crypto.encrypt(pk, _)).seq) + val jsonEncrypted = Json.toJson(toEncrypt.par.map(Crypto.encrypt(pks, _)).seq) println(jsonEncrypted) } } \ No newline at end of file diff --git a/app/utils/Crypto.scala b/app/utils/Crypto.scala index dcdd5600..5cb8f5a5 100644 --- a/app/utils/Crypto.scala +++ b/app/utils/Crypto.scala @@ -103,9 +103,11 @@ import java.math.BigInteger } /** encode and then encrypt a plaintext */ - def encrypt(pk: PublicKey, value: Long) = { - val encoded = encode(pk, value) - encryptEncoded(pk, encoded) + def encrypt(pks: Array[PublicKey], values: Array[Long]) = { + val encoded = pks.view.zipWithIndex.map( t => + encode(t._1, values(t._2)) + ).toArray + encryptEncoded(pks, encoded) } /** return a random bigint between 0 and max - 1 */ @@ -132,22 +134,30 @@ import java.math.BigInteger } /** encrypt an encoded plaintext */ - def encryptEncoded(pk: PublicKey, value: BigInt) = { - val r = randomBigInt(pk.q) - val alpha = pk.g.modPow(r, pk.p) - val beta = (value * (pk.y.modPow(r, pk.p))).mod(pk.p) - - // prove knowledge - val w = randomBigInt(pk.q) - - val commitment = pk.g.modPow(w, pk.p) - - val toHash = s"${alpha.toString}/${commitment.toString}" - val challenge = BigInt(sha256(toHash), 16) - - // compute response = w + randomness * challenge - val response = (w + (r * challenge)).mod(pk.q) - - EncryptedVote(Array(Choice(alpha, beta)), "now", Array(Popk(challenge, commitment, response))) + def encryptEncoded(pks: Array[PublicKey], values: Array[BigInt]) = { + var choices = Array[Choice]() + var proofs = Array[Popk]() + val evote = EncryptedVote(Array(), "now", Array()) + for ( index <- 0 to pks.length ) { + val pk = pks(index) + val value = values(index) + val r = randomBigInt(pk.q) + val alpha = pk.g.modPow(r, pk.p) + val beta = (value * (pk.y.modPow(r, pk.p))).mod(pk.p) + + // prove knowledge + val w = randomBigInt(pk.q) + + val commitment = pk.g.modPow(w, pk.p) + + val toHash = s"${alpha.toString}/${commitment.toString}" + val challenge = BigInt(sha256(toHash), 16) + + // compute response = w + randomness * challenge + val response = (w + (r * challenge)).mod(pk.q) + choices :+= Choice(alpha, beta) + proofs :+= Popk(challenge, commitment, response) + } + EncryptedVote(choices, "now", proofs) } } \ No newline at end of file From 12106dfd271a5dbc333824fcca1ad1b5bf2d7b24 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 7 Feb 2017 19:57:26 +0100 Subject: [PATCH 25/68] fix for length --- app/utils/Crypto.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/utils/Crypto.scala b/app/utils/Crypto.scala index 5cb8f5a5..f634cc4c 100644 --- a/app/utils/Crypto.scala +++ b/app/utils/Crypto.scala @@ -137,8 +137,7 @@ import java.math.BigInteger def encryptEncoded(pks: Array[PublicKey], values: Array[BigInt]) = { var choices = Array[Choice]() var proofs = Array[Popk]() - val evote = EncryptedVote(Array(), "now", Array()) - for ( index <- 0 to pks.length ) { + for ( index <- 0 until pks.length ) { val pk = pks(index) val value = values(index) val r = randomBigInt(pk.q) From c8b4c3e9c8eda21880e210910c626fa6afb9be63 Mon Sep 17 00:00:00 2001 From: Findeton Date: Wed, 8 Feb 2017 10:27:03 +0100 Subject: [PATCH 26/68] separate generate and send votes in 2 commands --- admin/admin.py | 120 ++++++++++++++++++++++++++++--------------------- 1 file changed, 70 insertions(+), 50 deletions(-) diff --git a/admin/admin.py b/admin/admin.py index 25f705de..1f575118 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -773,6 +773,62 @@ def gen_rnd_str(length, choices): for _ in range(length) ) + if args.vote_count <= 0: + raise Exception("vote count must be > 0") + + with TemporaryDirectory() as temp_path: + print("%s created temporary folder at %s" % (str(datetime.now()), temp_path)) + + election_id = cfg['election_id'] + vote_count = args.vote_count + cfg['encrypt-count'] = vote_count + + if cfg['ciphertexts'] is None: + raise Exception("missing ciphertexts argument") + + if cfg['plaintexts'] is None: + raise Exception("missing ciphertexts argument") + + _check_file(cfg['plaintexts']) + save_ciphertexts_path = cfg['ciphertexts'] + + # a list of base plaintexts to generate ballots + base_plaintexts_path = cfg["plaintexts"] + ciphertexts_path = os.path.join(temp_path, 'ciphertexts') + cfg['ciphertexts'] = ciphertexts_path + print("%s start generation of plaintexts" % str(datetime.now())) + cfg["plaintexts"] = gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count) + print("%s plaintexts created" % str(datetime.now())) + print("%s start dump_pks" % str(datetime.now())) + dump_pks(cfg, args) + print("%s pks dumped" % str(datetime.now())) + print("%s start ballot encryption" % str(datetime.now())) + start_time = time.time() + encrypt(cfg, args) + print("%s ballots encrypted" % str(datetime.now())) + end_time = time.time() + delta_t = end_time - start_time + troughput = vote_count / float(delta_t) + print("encrypted %i votes in %f secs. %f votes/sec" % (vote_count, delta_t, troughput)) + + shutil.copy2(ciphertexts_path, save_ciphertexts_path) + print("encrypted ballots saved to file %s" % ciphertexts_path) + +def send_votes(cfg, args): + def _open(path, mode): + return codecs.open(path, encoding='utf-8', mode=mode) + + def _check_file(path): + if not os.access(path, os.R_OK): + raise Exception("Error: can't read %s" % path) + if not os.path.isfile(path): + raise Exception("Error: not a file %s" % path) + + def _read_file(path): + _check_file(path) + with _open(path, mode='r') as f: + return f.read() + def gen_all_khmacs(vote_count, election_id): import hmac start_time = time.time() @@ -825,58 +881,21 @@ def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): if args.vote_count <= 0: raise Exception("vote count must be > 0") - with TemporaryDirectory() as temp_path: - print("%s created temporary folder at %s" % (str(datetime.now()), temp_path)) + if cfg['ciphertexts'] is None: + raise Exception("ciphertexts argument is missing") - election_id = cfg['election_id'] - vote_count = args.vote_count - cfg['encrypt-count'] = vote_count + _check_file(cfg['ciphertexts']) + ciphertexts_path = cfg['ciphertexts'] - save_ciphertexts = False - read_ciphertexts = False - if cfg['ciphertexts'] is not None: - # if the file exists, then read it: we don't need to generate it - if os.access(cfg['ciphertexts'], os.R_OK) and os.path.isfile(cfg['ciphertexts']): - read_ciphertexts = True - ciphertexts_path = cfg['ciphertexts'] - # if it doesn't exist, generate the ciphertexts and save them there - else: - _check_file(cfg['plaintexts']) - save_ciphertexts = True - save_ciphertexts_path = cfg['ciphertexts'] - else: - _check_file(cfg['plaintexts']) - - - if not read_ciphertexts: - # a list of base plaintexts to generate ballots - base_plaintexts_path = cfg["plaintexts"] - ciphertexts_path = os.path.join(temp_path, 'ciphertexts') - cfg['ciphertexts'] = ciphertexts_path - print("%s start generation of plaintexts" % str(datetime.now())) - cfg["plaintexts"] = gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count) - print("%s plaintexts created" % str(datetime.now())) - print("%s start dump_pks" % str(datetime.now())) - dump_pks(cfg, args) - print("%s pks dumped" % str(datetime.now())) - print("%s start ballot encryption" % str(datetime.now())) - start_time = time.time() - encrypt(cfg, args) - print("%s ballots encrypted" % str(datetime.now())) - end_time = time.time() - delta_t = end_time - start_time - troughput = vote_count / float(delta_t) - print("encrypted %i votes in %f secs. %f votes/sec" % (vote_count, delta_t, troughput)) - if save_ciphertexts: - shutil.copy2(ciphertexts_path, save_ciphertexts_path) - print("encrypted ballots saved to file %s" % ciphertexts_path) - else: - print("%s start khmac generation" % str(datetime.now())) - khmac_list = gen_all_khmacs(vote_count, election_id) - print("%s khmacs generated" % str(datetime.now())) - print("%s start sending ballots" % str(datetime.now())) - send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id) - print("%s ballots sent" % str(datetime.now())) + election_id = cfg['election_id'] + vote_count = args.vote_count + + print("%s start khmac generation" % str(datetime.now())) + khmac_list = gen_all_khmacs(vote_count, election_id) + print("%s khmacs generated" % str(datetime.now())) + print("%s start sending ballots" % str(datetime.now())) + send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id) + print("%s ballots sent" % str(datetime.now())) def get_hmac(cfg, userId, objType, objId, perm): import hmac @@ -921,6 +940,7 @@ def main(argv): change_social : changes the social netoworks share buttons configuration authapi_ensure_acls --acls-path : ensure that the acls inside acl_path exist. gen_votes : generate votes +send_votes : send votes generated by the command gen_votes ''') parser.add_argument('--ciphertexts', help='file to write ciphertetxs (used in dump, load and encrypt)') parser.add_argument('--acls-path', help='''the file has one line per acl with format: '(email:email@example.com|tlf:+34666777888),permission_name,object_type,object_id,user_election_id' ''') From 13839b5b622bbacf5e7160b07169c7283265ba8f Mon Sep 17 00:00:00 2001 From: Findeton Date: Wed, 8 Feb 2017 10:38:38 +0100 Subject: [PATCH 27/68] move gen_rnd_str function --- admin/admin.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/admin/admin.py b/admin/admin.py index 1f575118..9a18b3d4 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -767,12 +767,6 @@ def gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count): _write_file(new_plaintext_path, new_plaintext) return new_plaintext_path - def gen_rnd_str(length, choices): - return ''.join( - random.SystemRandom().choice(choices) - for _ in range(length) - ) - if args.vote_count <= 0: raise Exception("vote count must be > 0") @@ -829,6 +823,12 @@ def _read_file(path): with _open(path, mode='r') as f: return f.read() + def gen_rnd_str(length, choices): + return ''.join( + random.SystemRandom().choice(choices) + for _ in range(length) + ) + def gen_all_khmacs(vote_count, election_id): import hmac start_time = time.time() From 2b17ceb0e5a8fd6e9a2e97adca16322172c0b375 Mon Sep 17 00:00:00 2001 From: Findeton Date: Wed, 8 Feb 2017 15:49:38 +0100 Subject: [PATCH 28/68] add vote hash to vote callback on headers --- app/controllers/BallotboxApi.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/controllers/BallotboxApi.scala b/app/controllers/BallotboxApi.scala index e73c4512..9900b90e 100644 --- a/app/controllers/BallotboxApi.scala +++ b/app/controllers/BallotboxApi.scala @@ -95,7 +95,8 @@ object BallotboxApi extends Controller with Response { .replace("${eid}", electionId+"") .replace("${uid}", voterId) , - message + message, + vote.vote_hash ) } Ok(response(result)) @@ -197,7 +198,7 @@ object BallotboxApi extends Controller with Response { } } - private def postVoteCallback(url: String, message: String) = { + private def postVoteCallback(url: String, message: String, vote_hash: String) = { try { println(s"posting to $url") val hmac = Crypto.hmac(boothSecret, message) @@ -205,7 +206,8 @@ object BallotboxApi extends Controller with Response { val f = WS.url(url) .withHeaders( "Accept" -> "application/json", - "Authorization" -> khmac) + "Authorization" -> khmac, + "BallotTracker" -> vote_hash) .post(Results.EmptyContent()) .map { resp => if(resp.status != HTTP.ACCEPTED) { From fd472ab951bd2693df12d400b0e8307cb6d7a7d9 Mon Sep 17 00:00:00 2001 From: Eduardo Robles Elvira Date: Fri, 10 Feb 2017 01:45:40 +0100 Subject: [PATCH 29/68] adding tally download timeout --- app/controllers/ElectionsApi.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/ElectionsApi.scala b/app/controllers/ElectionsApi.scala index 0fcecb0d..19ba2a52 100644 --- a/app/controllers/ElectionsApi.scala +++ b/app/controllers/ElectionsApi.scala @@ -718,7 +718,8 @@ object ElectionsApi // taken from https://www.playframework.com/documentation/2.3.x/ScalaWS val futureResponse: Future[(WSResponseHeaders, Enumerator[Array[Byte]])] = - WS.url(url).getStream() + /* 3600 seconds timeout should be enough for everybody WCPGR? */ + WS.url(url).withRequestTimeout(3600000).getStream() val downloadedFile: Future[Unit] = futureResponse.flatMap { case (headers, body) => From 8e1b12ceb57e8beaa77631e126a46aef17ea1995 Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 16 Feb 2017 13:17:01 +0100 Subject: [PATCH 30/68] add fast vote index db evolution --- conf/evolutions/default/6.sql | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 conf/evolutions/default/6.sql diff --git a/conf/evolutions/default/6.sql b/conf/evolutions/default/6.sql new file mode 100644 index 00000000..cd376277 --- /dev/null +++ b/conf/evolutions/default/6.sql @@ -0,0 +1,7 @@ +# --- !Ups + +CREATE INDEX "voter_id_created_desc_idx" ON vote (voter_id asc, created desc); + +# --- !Downs + +DROP INDEX "voter_id_created_desc_idx"; \ No newline at end of file From a3ced8d8205b0686789938554073b1fb73d52bd5 Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 16 Feb 2017 15:57:41 +0100 Subject: [PATCH 31/68] don't call dump_votes_with_ids on tally-with-ids as it shoudln't be required anymore --- admin/batch.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/admin/batch.py b/admin/batch.py index 6fd16076..a506beed 100644 --- a/admin/batch.py +++ b/admin/batch.py @@ -104,12 +104,6 @@ def main(argv): elif args.command == 'list-tally-with-ids': for eid in args.election_ids: - print('next id %d, dumping votes with matching ids (in private datastore)' % eid) - ret = cycle.dump_votes_with_ids(eid) - if ret in [400, 500]: - print("dump_votes_with_ids returned %d, continuing without it" % ret) - continue - print('next id %d, stopping election' % eid) ret = cycle.stop(eid) if ret in [400, 500]: @@ -223,12 +217,6 @@ def main(argv): else: next_id = cfg['id'] - print('next id %d, dumping votes with matching ids (in private datastore)' % next_id) - ret = cycle.dump_votes_with_ids(next_id) - if ret in [400, 500]: - print("dump_votes_with_ids returned %d, continuing without it" % ret) - continue - print('next id %d, stopping election' % next_id) ret = cycle.stop(next_id) if ret in [400, 500]: From 57391769dc28baadfcf7fd15c669cc37a4a56953 Mon Sep 17 00:00:00 2001 From: Findeton Date: Fri, 3 Mar 2017 10:23:00 +0100 Subject: [PATCH 32/68] don't check real --- app/controllers/ElectionsApi.scala | 40 +----------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/app/controllers/ElectionsApi.scala b/app/controllers/ElectionsApi.scala index 4e035674..007f0018 100644 --- a/app/controllers/ElectionsApi.scala +++ b/app/controllers/ElectionsApi.scala @@ -89,7 +89,7 @@ object ElectionsApi def update(id: Long) = HAction("", "AuthEvent", id, "edit|update").async(BodyParsers.parse.json) { request => updateElection(id, request) } - + /** updates an election's social share buttons config */ def updateShare(id: Long) = HAction("", "AuthEvent", id, "edit|update-share").async(BodyParsers.parse.json) { request => updateShareElection(id, request) @@ -487,25 +487,6 @@ object ElectionsApi { try { val validated = config.validate(authorities, id) - if (validated.real) - { - request - .headers - .get("Authorization") - .map( - HMACAuthAction("", "AuthEvent", id, "edit|register-real") - .validate(request) - ) match - { - case Some(true) => None - case _ => BadRequest( - error( - s"Invalid config json, user has not permissions to " + - s"register real elections" - ) - ) - } - } DB.withSession { implicit session => @@ -618,25 +599,6 @@ object ElectionsApi config => { val validated = config.validate(authorities, id) - if (validated.real) - { - request - .headers - .get("Authorization") - .map( - HMACAuthAction("", "AuthEvent", id, "edit|register-real") - .validate(request) - ) match - { - case Some(true) => None - case _ => BadRequest( - error( - s"Invalid config json, user has not permissions to " + - s"register real elections" - ) - ) - } - } val result = DAL.elections.updateConfig(id, validated.asString, validated.start_date, validated.end_date) Ok(response(result)) From 1c3665c0c3d062ac8bc2a0b865d803cd1e6aca55 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 14 Mar 2017 15:14:34 +0100 Subject: [PATCH 33/68] adding code --- admin/console.sh | 21 +++ app/commands/Console.scala | 289 +++++++++++++++++++++++++++++++++++++ 2 files changed, 310 insertions(+) create mode 100755 admin/console.sh create mode 100644 app/commands/Console.scala diff --git a/admin/console.sh b/admin/console.sh new file mode 100755 index 00000000..ffd88f50 --- /dev/null +++ b/admin/console.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +# This file is part of agora_elections. +# Copyright (C) 2014-2016 Agora Voting SL + +# agora_elections is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License. + +# agora_elections is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with agora_elections. If not, see . + +IVY=~/.ivy2/cache +AGORA_HOME=.. +java -classpath $IVY/cache/com.typesafe.akka/akka-actor_2.11/jars/akka-actor_2.11-2.3.4.jar:$AGORA_HOME/target/scala-2.11/agora-elections_2.11-1.0-SNAPSHOT.jar:$IVY/com.typesafe.play/play-json_2.11/jars/play-json_2.11-2.3.6.jar:$IVY/org.scala-lang/scala-library/jars/scala-library-2.11.1.jar:$IVY/com.fasterxml.jackson.core/jackson-core/bundles/jackson-core-2.3.3.jar:$IVY/com.fasterxml.jackson.core/jackson-databind/bundles/jackson-databind-2.3.3.jar:$IVY/com.fasterxml.jackson.core/jackson-annotations/bundles/jackson-annotations-2.3.2.jar:$IVY/com.typesafe.play/play-functional_2.11/jars/play-functional_2.11-2.3.6.jar:$IVY/org.joda/joda-convert/jars/joda-convert-1.6.jar:$IVY/joda-time/joda-time/jars/joda-time-2.3.jar commands.Console $* + diff --git a/app/commands/Console.scala b/app/commands/Console.scala new file mode 100644 index 00000000..5e08d807 --- /dev/null +++ b/app/commands/Console.scala @@ -0,0 +1,289 @@ +/** + * This file is part of agora_elections. + * Copyright (C) 2017 Agora Voting SL + + * agora_elections is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License. + + * agora_elections is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + + * You should have received a copy of the GNU Affero General Public License + * along with agora_elections. If not, see . +**/ +package commands + +import models._ +import utils.JsonFormatters._ +import utils.Crypto + +import java.util.concurrent.Executors +import scala.concurrent._ + +import javax.crypto.spec.SecretKeySpec +import javax.crypto.Mac +import javax.xml.bind.DatatypeConverter +import java.math.BigInteger +import scala.concurrent.forkjoin._ +import scala.collection.mutable.{ListBuffer, ArrayBuffer} + +import java.nio.file.{Paths, Files} + +case class Answer(options: Array[Int] = Array[Int]()) +// id is the election ID +case class PlaintextBallot(id: Int = -1, answers: Array[Answer] = Array[Answer]()) +case class PlaintextError(message: String) extends Exception(message) +case class DumpPksError(message: String) extends Exception(message) + +object PlaintextBallot { + val ID = 0 +} + +/** + * Command for admin purposes + * + * use runMain commands.Command from console + * or + * activator "run-main commands.Command " + * from CLI + */ +object Console { + implicit val ec = ExecutionContext.fromExecutor(new ForkJoinPool(100)) + + var vote_count = 0 + var plaintexts_path = "plaintexts.txt" + var ciphertexts_path = "ciphertexts.csv" + var shared_secret = "" + var datastore = "/home/agoraelections/datastore" + var batch_size = 50 + + private def parse_args(args: Array[String]) = { + var arg_index = 0 + while (arg_index + 2 < args.length) { + // number of ballots to create + if ("--vote-count" == args(arg_index + 1)) { + vote_count = args(arg_index + 2).toInt + arg_index += 2 + } + else if ("--plaintexts" == args(arg_index + 1)) { + plaintexts_path = args(arg_index + 2) + arg_index += 2 + } else if ("--ciphertexts" == args(arg_index + 1)) { + ciphertexts_path = args(arg_index + 2) + arg_index += 2 + } else if ("--shared-secret" == args(arg_index + 1)) { + shared_secret = args(arg_index + 2) + arg_index += 2 + } else if ("--batch-size" == args(arg_index + 1)) { + batch_size = args(arg_index + 2).toInt + arg_index += 2 + } + } else { + throw new java.lang.IllegalArgumentException("unrecognized argument: " + args(arg_index + 1)) + } + } + } + + private def showHelp() = { + System.out.println("showHelp") + () + } + + private def dump_pks(electionId: Int): Future[Unit] = { + } + + private def processPlaintextLine(line: String, lineNumber: Int) : PlaintextBallot = { + var strIndex: Option[String] = None + var state = PlaintextBallot.ID + val ballot = PlaintextBallot() + var optionsBuffer: Option[ArrayBuffer[Int]] = None + var answersBuffer: ArrayBuffer[Answer] = ArrayBuffer[Answer]() + for (int i = 0; i < line.length; i++) { + val c = line.charAt(i) + if(c.isDigit) { // keep reading digits till we get the whole number + strIndex match { + // add a character to the string containing a number + case Some(strIndexValue) => + strIndex = Some(strIndexValue + c.toString) + case None => () + // add the first character to the string containing a number + strIndex = Some(c.toString) + } + } else { + if (PlaintextBallot.ID == state) { + if ('|' != c) { + throw PlaintextError(s"Error on line $lineNumber, character $i: character separator '|' not found after election index . Line: $line") + } + strIndex match { + case Some(strIndexValue) => + ballot.id = strIndex.get.toInt + strIndex = None + optionsBuffer = Some(ArrayBuffer[Int]()) + state = PlaintextBallot.ANSWER + case None => + throw PlaintextError(s"Error on line $lineNumber, character $i: election index not recognized. Line: $line") + } + } else if (PlaintextBallot.ANSWER == state) { + optionsBuffer match { + case Some(optionsBufferValue) => + if ('|' == c) { + if (strIndex.isDefined) { + optionsBufferValue += strIndex.get.toInt + strIndex = None + } + answersBuffer += Answer(optionsBufferValue.toArray) + optionsBuffer = Some(ArrayBuffer[Int]()) + } else if(',' == c) { + strIndex match { + case Some(strIndexValue) => + optionsBufferValue += strIndexValue.toInt + strIndex = None + case None => + throw PlaintextError(s"Error on line $lineNumber, character $i: option number not recognized before comma on question ${ballot.answers.length}. Line: $line") + } + } else { + throw PlaintextError(s"Error on line $lineNumber, character $i: invalid character: $c. Line: $line") + } + case None => + PlaintextError(s"Error on line $lineNumber, character $i: unknown error, invalid state. Line: $line") + } + } + } + } + // add the last Answer + optionsBuffer match { + case Some(optionsBufferValue) => + answersBuffer += Answer(optionsBufferValue.toArray) + case None => + throw PlaintextError(s"Error on line $lineNumber: unknown error, invalid state. Line: $line") + } + ballot.answers = answersBuffer.toArray + ballot + } + + private def parsePlaintexts(): Future[(scala.collection.immutable.List[PlaintextBallot], scala.collection.immutable.Set[Int])] = { + val promise = Promise[(scala.collection.immutable.List[PlaintextBallot], scala.collection.immutable.Set[Int])]() + Future { + val ballotsList = scala.collection.mutable.ListBuffer[PlaintextBallot]() + val electionsSet = scala.collection.mutable.Set[Int]() + if (Files.exists(Paths.get(plaintexts_path))) { + val fileLinesMap: List[PlaintextBallot] = io.Source.fromFile(plaintexts_path).getLines() foreach { + line => + val ballot = processPlaintextLine(line) + list += ballot + electionsSet += ballot.id + } + } else { + throw new java.io.FileNotFoundException("tally does not exist") + } + promise success ( ballotsList.sortBy(_.id).toList, electionsSet.toSet ) + } (ec) onFailure { case error => + promise failure error + } (ec) + } + + private def dump_pks_elections(electionsSet: scala.collection.immutable.Set[Int]): Future[Unit] = { + val promise = Promise[Unit]() + Future { + var count : Int = 0 + var futuresCreated = Promise[Unit]() + futuresCreated onFailure { case error => + promise failure error + } + for (electionId <- electionsSet) { + count.synchronized { + count += 1 + } + dump_pks(electionId) onComplete { + case Success(value) => + futuresCreated onSuccess { case value2 => + count.synchronized { + if ( 0 >= count ) { + promise failure DumpPksError("Logic error") + } + count -= 1 + if ( 0 == count ) { + promise success () + } + } + } (ec) + case Failure(error) => + futuresCreated failure error + } (ec) + } + futuresCreated success () + } (ec) onFailure { case error => + promise failure error + } + promise.future + } + + private def encryptBallotTask(ballotsList: scala.collection.immutable.List[PlaintextBallot], index: Int, numBallots: Int, fileWriteMutex: Unit) : Future[Unit] = { + val promise = Promise[Unit]() + Future { + promise success () + } (ec) onFailure { case error => + promise failure error + } (ec) + promise.future + } + + private def encryptBallots(ballotsList: scala.collection.immutable.List[PlaintextBallot], electionsSet: scala.collection.immutable.Set[Int]): Future[Unit] = { + val promise = Promise[Unit]() + Future { + val fileWriteMutex: Unit = () + val count : Int = 0 + while ( count < vote_count ) { + val numBallots : Int = if ( vote_count - count < batch_size ) { + vote_count - count + } else { + batch_size + } + encryptBallotTask(ballotsList, count % ballotsList.length, numBallots, fileWriteMutex) + count += numBallots + } + } (ec) onFailure { case error => + promise failure error + } (ec) + promise.future + } + + private def gen_votes(): Future[Unit] = { + val promise = Promise[Unit]() + Future { + promise completeWith parsePlaintexts() flatMap { (ballotsList, electionsSet) => + dump_pks_elections(electionsSet) flatMap { u => + encryptBallots(ballotsList, electionsSet) + } (ec) + } (ec) + } (ec) onFailure { case error => + promise failure error + } (ec) + promise.future + } + + private def send_votes() = { + } + + def main(args: Array[String]) = { + System.out.println("hi there") + if(0 == args.length) { + showHelp() + } else { + parse_args(args) + val command = args(0) + if ("gen_votes" == command) { + gen_votes() onSuccess { case a => + System.out.println("hi there") + } + } else if ( "send_votes" == command) { + send_votes() + } else { + showHelp() + } + } + } +} \ No newline at end of file From da206d879c8829fe780414467eddcabf88a53631 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 14 Mar 2017 16:23:57 +0100 Subject: [PATCH 34/68] fixes build --- app/commands/Console.scala | 91 +++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 5e08d807..79dbeb44 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -29,6 +29,7 @@ import javax.xml.bind.DatatypeConverter import java.math.BigInteger import scala.concurrent.forkjoin._ import scala.collection.mutable.{ListBuffer, ArrayBuffer} +import scala.util.{Success, Failure} import java.nio.file.{Paths, Files} @@ -40,6 +41,7 @@ case class DumpPksError(message: String) extends Exception(message) object PlaintextBallot { val ID = 0 + val ANSWER = 1 } /** @@ -80,7 +82,6 @@ object Console { } else if ("--batch-size" == args(arg_index + 1)) { batch_size = args(arg_index + 2).toInt arg_index += 2 - } } else { throw new java.lang.IllegalArgumentException("unrecognized argument: " + args(arg_index + 1)) } @@ -93,15 +94,23 @@ object Console { } private def dump_pks(electionId: Int): Future[Unit] = { + val promise = Promise[Unit]() + Future { + promise success ( () ) + } onComplete { case Failure(error) => + promise failure error + case _ => + } + promise.future } private def processPlaintextLine(line: String, lineNumber: Int) : PlaintextBallot = { var strIndex: Option[String] = None var state = PlaintextBallot.ID - val ballot = PlaintextBallot() + var ballot = PlaintextBallot() var optionsBuffer: Option[ArrayBuffer[Int]] = None var answersBuffer: ArrayBuffer[Answer] = ArrayBuffer[Answer]() - for (int i = 0; i < line.length; i++) { + for (i <- 0 until line.length) { val c = line.charAt(i) if(c.isDigit) { // keep reading digits till we get the whole number strIndex match { @@ -119,7 +128,7 @@ object Console { } strIndex match { case Some(strIndexValue) => - ballot.id = strIndex.get.toInt + ballot = PlaintextBallot(strIndex.get.toInt, ballot.answers) strIndex = None optionsBuffer = Some(ArrayBuffer[Int]()) state = PlaintextBallot.ANSWER @@ -160,7 +169,7 @@ object Console { case None => throw PlaintextError(s"Error on line $lineNumber: unknown error, invalid state. Line: $line") } - ballot.answers = answersBuffer.toArray + ballot = PlaintextBallot(ballot.id, answersBuffer.toArray) ballot } @@ -170,53 +179,59 @@ object Console { val ballotsList = scala.collection.mutable.ListBuffer[PlaintextBallot]() val electionsSet = scala.collection.mutable.Set[Int]() if (Files.exists(Paths.get(plaintexts_path))) { - val fileLinesMap: List[PlaintextBallot] = io.Source.fromFile(plaintexts_path).getLines() foreach { - line => - val ballot = processPlaintextLine(line) - list += ballot - electionsSet += ballot.id + io.Source.fromFile(plaintexts_path).getLines().zipWithIndex.foreach { + case (line, number) => + val ballot = processPlaintextLine(line, number) + ballotsList += ballot + electionsSet += ballot.id } } else { throw new java.io.FileNotFoundException("tally does not exist") } promise success ( ballotsList.sortBy(_.id).toList, electionsSet.toSet ) - } (ec) onFailure { case error => + } onComplete { case Failure(error) => promise failure error - } (ec) + case _ => + } + promise.future } - + private def dump_pks_elections(electionsSet: scala.collection.immutable.Set[Int]): Future[Unit] = { val promise = Promise[Unit]() Future { var count : Int = 0 var futuresCreated = Promise[Unit]() - futuresCreated onFailure { case error => - promise failure error + futuresCreated.future onComplete { + case Failure(error) => + promise failure error + case _ => } for (electionId <- electionsSet) { - count.synchronized { + this.synchronized { count += 1 } dump_pks(electionId) onComplete { case Success(value) => - futuresCreated onSuccess { case value2 => - count.synchronized { + futuresCreated.future onComplete { case Success(value2) => + this.synchronized { if ( 0 >= count ) { promise failure DumpPksError("Logic error") } count -= 1 if ( 0 == count ) { - promise success () + promise success ( () ) } } - } (ec) + case _ => + } case Failure(error) => futuresCreated failure error - } (ec) + } } - futuresCreated success () - } (ec) onFailure { case error => + futuresCreated success ( () ) + } onComplete { case Failure(error) => promise failure error + case _ => } promise.future } @@ -224,10 +239,11 @@ object Console { private def encryptBallotTask(ballotsList: scala.collection.immutable.List[PlaintextBallot], index: Int, numBallots: Int, fileWriteMutex: Unit) : Future[Unit] = { val promise = Promise[Unit]() Future { - promise success () - } (ec) onFailure { case error => + promise success ( () ) + } onComplete { case Failure(error) => promise failure error - } (ec) + case _ => + } promise.future } @@ -235,7 +251,7 @@ object Console { val promise = Promise[Unit]() Future { val fileWriteMutex: Unit = () - val count : Int = 0 + var count : Int = 0 while ( count < vote_count ) { val numBallots : Int = if ( vote_count - count < batch_size ) { vote_count - count @@ -245,23 +261,26 @@ object Console { encryptBallotTask(ballotsList, count % ballotsList.length, numBallots, fileWriteMutex) count += numBallots } - } (ec) onFailure { case error => + } onComplete { case Failure(error) => promise failure error - } (ec) + case _ => + } promise.future } private def gen_votes(): Future[Unit] = { val promise = Promise[Unit]() Future { - promise completeWith parsePlaintexts() flatMap { (ballotsList, electionsSet) => - dump_pks_elections(electionsSet) flatMap { u => - encryptBallots(ballotsList, electionsSet) - } (ec) - } (ec) - } (ec) onFailure { case error => + promise completeWith { parsePlaintexts() flatMap { case (ballotsList, electionsSet) => + dump_pks_elections(electionsSet) flatMap { case u => + encryptBallots(ballotsList, electionsSet) + } + } + } + } onComplete { case Failure(error) => promise failure error - } (ec) + case _ => + } promise.future } From 81076441587e77904953243e144a81429110bf8a Mon Sep 17 00:00:00 2001 From: Findeton Date: Wed, 15 Mar 2017 13:34:19 +0100 Subject: [PATCH 35/68] more code --- app/commands/Console.scala | 210 ++++++++++++++++++++++++------------- 1 file changed, 136 insertions(+), 74 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 79dbeb44..68ad0854 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -28,16 +28,28 @@ import javax.crypto.Mac import javax.xml.bind.DatatypeConverter import java.math.BigInteger import scala.concurrent.forkjoin._ -import scala.collection.mutable.{ListBuffer, ArrayBuffer} -import scala.util.{Success, Failure} +import scala.collection.mutable.ArrayBuffer +import scala.util.{Try, Success, Failure} + +import play.api.libs.concurrent.Execution.Implicits.defaultContext +import play.libs.Akka import java.nio.file.{Paths, Files} -case class Answer(options: Array[Int] = Array[Int]()) +import play.api.Play.current +import play.api.libs.ws._ +import play.api.libs.ws.ning.NingAsyncHttpClientConfigBuilder +import play.api.libs.ws.ning.NingWSClient +import play.api.mvc._ +import play.api.http.{Status => HTTP} + + +case class Answer(options: Array[Long] = Array[Long]()) // id is the election ID -case class PlaintextBallot(id: Int = -1, answers: Array[Answer] = Array[Answer]()) +case class PlaintextBallot(id: Long = -1, answers: Array[Answer] = Array[Answer]()) case class PlaintextError(message: String) extends Exception(message) case class DumpPksError(message: String) extends Exception(message) +case class BallotEncryptionError(message: String) extends Exception(message) object PlaintextBallot { val ID = 0 @@ -53,21 +65,33 @@ object PlaintextBallot { * from CLI */ object Console { - implicit val ec = ExecutionContext.fromExecutor(new ForkJoinPool(100)) + //implicit val ec = ExecutionContext.fromExecutor(new ForkJoinPool(100)) + //implicit val slickExecutionContext = Akka.system.dispatchers.lookup("play.akka.actor.slick-context") - var vote_count = 0 + var vote_count : Long = 0 + var host = "localhost" + var port : Long = 9000 var plaintexts_path = "plaintexts.txt" var ciphertexts_path = "ciphertexts.csv" var shared_secret = "" var datastore = "/home/agoraelections/datastore" - var batch_size = 50 + var batch_size : Long = 50 + + // copied from https://www.playframework.com/documentation/2.3.x/ScalaWS + val clientConfig = new DefaultWSClientConfig() + val secureDefaults:com.ning.http.client.AsyncHttpClientConfig = new NingAsyncHttpClientConfigBuilder(clientConfig).build() + val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder(secureDefaults) + builder.setCompressionEnabled(true) + val secureDefaultsWithSpecificOptions:com.ning.http.client.AsyncHttpClientConfig = builder.build() + implicit val wsClient = new play.api.libs.ws.ning.NingWSClient(secureDefaultsWithSpecificOptions) + private def parse_args(args: Array[String]) = { var arg_index = 0 while (arg_index + 2 < args.length) { // number of ballots to create if ("--vote-count" == args(arg_index + 1)) { - vote_count = args(arg_index + 2).toInt + vote_count = args(arg_index + 2).toLong arg_index += 2 } else if ("--plaintexts" == args(arg_index + 1)) { @@ -80,35 +104,29 @@ object Console { shared_secret = args(arg_index + 2) arg_index += 2 } else if ("--batch-size" == args(arg_index + 1)) { - batch_size = args(arg_index + 2).toInt + batch_size = args(arg_index + 2).toLong + arg_index += 2 + } else if ("--port" == args(arg_index + 1)) { + port = args(arg_index + 2).toLong + if (port <= 0 || port > 65535) { + throw new java.lang.IllegalArgumentException(s"Invalid port $port") + } arg_index += 2 } else { - throw new java.lang.IllegalArgumentException("unrecognized argument: " + args(arg_index + 1)) + throw new java.lang.IllegalArgumentException("Unrecognized argument: " + args(arg_index + 1)) } } } private def showHelp() = { System.out.println("showHelp") - () } - private def dump_pks(electionId: Int): Future[Unit] = { - val promise = Promise[Unit]() - Future { - promise success ( () ) - } onComplete { case Failure(error) => - promise failure error - case _ => - } - promise.future - } - - private def processPlaintextLine(line: String, lineNumber: Int) : PlaintextBallot = { + private def processPlaintextLine(line: String, lineNumber: Long) : PlaintextBallot = { var strIndex: Option[String] = None var state = PlaintextBallot.ID var ballot = PlaintextBallot() - var optionsBuffer: Option[ArrayBuffer[Int]] = None + var optionsBuffer: Option[ArrayBuffer[Long]] = None var answersBuffer: ArrayBuffer[Answer] = ArrayBuffer[Answer]() for (i <- 0 until line.length) { val c = line.charAt(i) @@ -128,9 +146,9 @@ object Console { } strIndex match { case Some(strIndexValue) => - ballot = PlaintextBallot(strIndex.get.toInt, ballot.answers) + ballot = PlaintextBallot(strIndex.get.toLong, ballot.answers) strIndex = None - optionsBuffer = Some(ArrayBuffer[Int]()) + optionsBuffer = Some(ArrayBuffer[Long]()) state = PlaintextBallot.ANSWER case None => throw PlaintextError(s"Error on line $lineNumber, character $i: election index not recognized. Line: $line") @@ -140,15 +158,15 @@ object Console { case Some(optionsBufferValue) => if ('|' == c) { if (strIndex.isDefined) { - optionsBufferValue += strIndex.get.toInt + optionsBufferValue += strIndex.get.toLong strIndex = None } answersBuffer += Answer(optionsBufferValue.toArray) - optionsBuffer = Some(ArrayBuffer[Int]()) + optionsBuffer = Some(ArrayBuffer[Long]()) } else if(',' == c) { strIndex match { case Some(strIndexValue) => - optionsBufferValue += strIndexValue.toInt + optionsBufferValue += strIndexValue.toLong strIndex = None case None => throw PlaintextError(s"Error on line $lineNumber, character $i: option number not recognized before comma on question ${ballot.answers.length}. Line: $line") @@ -173,97 +191,138 @@ object Console { ballot } - private def parsePlaintexts(): Future[(scala.collection.immutable.List[PlaintextBallot], scala.collection.immutable.Set[Int])] = { - val promise = Promise[(scala.collection.immutable.List[PlaintextBallot], scala.collection.immutable.Set[Int])]() + private def parsePlaintexts(): Future[(scala.collection.immutable.List[PlaintextBallot], scala.collection.immutable.Set[Long])] = { + val promise = Promise[(scala.collection.immutable.List[PlaintextBallot], scala.collection.immutable.Set[Long])]() Future { - val ballotsList = scala.collection.mutable.ListBuffer[PlaintextBallot]() - val electionsSet = scala.collection.mutable.Set[Int]() if (Files.exists(Paths.get(plaintexts_path))) { + val ballotsList = scala.collection.mutable.ListBuffer[PlaintextBallot]() + val electionsSet = scala.collection.mutable.LinkedHashSet[Long]() io.Source.fromFile(plaintexts_path).getLines().zipWithIndex.foreach { case (line, number) => val ballot = processPlaintextLine(line, number) ballotsList += ballot electionsSet += ballot.id } + if ( electionsSet.isEmpty || ballotsList.isEmpty ) { + throw PlaintextError("Error: no ballot found") + } else { + promise success ( ( ballotsList.sortBy(_.id).toList, electionsSet.toSet ) ) + } } else { throw new java.io.FileNotFoundException("tally does not exist") } - promise success ( ballotsList.sortBy(_.id).toList, electionsSet.toSet ) - } onComplete { case Failure(error) => + } recover { case error: Throwable => promise failure error - case _ => } promise.future } - private def dump_pks_elections(electionsSet: scala.collection.immutable.Set[Int]): Future[Unit] = { + private def get_khmac(userId: String, objType: String, objId: Long, perm: String) : String = { + val now: Long = System.currentTimeMillis / 1000 + val message = s"$userId:$objType:$objId:$perm:$now" + val hmac = Crypto.hmac(shared_secret, message) + val khmac = s"khmac:///sha-256;$hmac/$message" + khmac + } + + private def dump_pks(electionId: Long): Future[Unit] = { val promise = Promise[Unit]() Future { - var count : Int = 0 - var futuresCreated = Promise[Unit]() - futuresCreated.future onComplete { - case Failure(error) => - promise failure error - case _ => + val auth = get_khmac("", "AuthEvent", electionId, "edit") + val url = s"http://$host:$port/api/election/$electionId/dump-pks" + wsClient.url(url) + .withHeaders("Authorization" -> auth) + .post(Results.EmptyContent()).map { response => + if(response.status == HTTP.ACCEPTED) { + promise success ( () ) + } else { + promise failure DumpPksError(s"HTTP POST request to $url returned status: ${response.status} and body: ${response.body}") + } + } recover { case error: Throwable => + promise failure error } + } recover { case error: Throwable => + promise failure error + } + promise.future + } + + private def dump_pks_elections(electionsSet: scala.collection.immutable.Set[Long]): Future[Unit] = { + val promise = Promise[Unit]() + Future { + var count : Long = 0 + var futuresCreated = Promise[Unit]() for (electionId <- electionsSet) { - this.synchronized { - count += 1 - } - dump_pks(electionId) onComplete { - case Success(value) => - futuresCreated.future onComplete { case Success(value2) => + count += 1 + dump_pks(electionId) flatMap { + value => + futuresCreated.future map { value2 => this.synchronized { if ( 0 >= count ) { - promise failure DumpPksError("Logic error") + throw DumpPksError("Logic error") } count -= 1 if ( 0 == count ) { promise success ( () ) } } - case _ => } - case Failure(error) => - futuresCreated failure error + } recover { + case error: Throwable => + promise failure error } } futuresCreated success ( () ) - } onComplete { case Failure(error) => + } recover { case error: Throwable => promise failure error - case _ => } promise.future } - private def encryptBallotTask(ballotsList: scala.collection.immutable.List[PlaintextBallot], index: Int, numBallots: Int, fileWriteMutex: Unit) : Future[Unit] = { + private def encryptBallotTask(ballotsList: scala.collection.immutable.List[PlaintextBallot], index: Long, numBallots: Long) : Future[Unit] = { val promise = Promise[Unit]() Future { promise success ( () ) - } onComplete { case Failure(error) => + } recover { case error: Throwable => promise failure error - case _ => } promise.future } - private def encryptBallots(ballotsList: scala.collection.immutable.List[PlaintextBallot], electionsSet: scala.collection.immutable.Set[Int]): Future[Unit] = { + private def encryptBallots(ballotsList: scala.collection.immutable.List[PlaintextBallot], electionsSet: scala.collection.immutable.Set[Long]): Future[Unit] = { val promise = Promise[Unit]() Future { - val fileWriteMutex: Unit = () - var count : Int = 0 - while ( count < vote_count ) { - val numBallots : Int = if ( vote_count - count < batch_size ) { - vote_count - count - } else { + var taskCounter : Long = 0 + var ballotCounter : Long = 0 + val tasksCreated = Promise[Unit]() + while (ballotCounter < vote_count) { + taskCounter += 1 + val numBallots = if (vote_count > batch_size + ballotCounter) { batch_size + } else { + vote_count - ballotCounter + } + encryptBallotTask (ballotsList, ballotCounter % ballotsList.length, numBallots) flatMap { value => + tasksCreated.future map { value2 => + this.synchronized { + if ( 0 >= taskCounter ) { + throw BallotEncryptionError("Logic error") + } + taskCounter -= 1 + if ( 0 == taskCounter ) { + promise success ( () ) + } + } + } + } recover { + case error: Throwable => + promise failure error } - encryptBallotTask(ballotsList, count % ballotsList.length, numBallots, fileWriteMutex) - count += numBallots + ballotCounter += numBallots } - } onComplete { case Failure(error) => + tasksCreated success ( () ) + } recover { case error: Throwable => promise failure error - case _ => } promise.future } @@ -277,9 +336,8 @@ object Console { } } } - } onComplete { case Failure(error) => + } recover { case error: Throwable => promise failure error - case _ => } promise.future } @@ -288,15 +346,19 @@ object Console { } def main(args: Array[String]) = { - System.out.println("hi there") if(0 == args.length) { showHelp() } else { parse_args(args) val command = args(0) if ("gen_votes" == command) { - gen_votes() onSuccess { case a => - System.out.println("hi there") + gen_votes() onComplete { + case Success(value) => + println("gen_votes success") + System.exit(0) + case Failure(error) => + println("gen_votes error " + error) + System.exit(-1) } } else if ( "send_votes" == command) { send_votes() From 6324df67aac8b2c6f2c5b32a7873b11451c59f71 Mon Sep 17 00:00:00 2001 From: Findeton Date: Wed, 15 Mar 2017 17:29:06 +0100 Subject: [PATCH 36/68] add encrypt --- app/commands/Console.scala | 146 +++++++++++++++++++++---------------- 1 file changed, 85 insertions(+), 61 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 68ad0854..1ba43efc 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -34,6 +34,8 @@ import scala.util.{Try, Success, Failure} import play.api.libs.concurrent.Execution.Implicits.defaultContext import play.libs.Akka +import play.api.libs.json._ + import java.nio.file.{Paths, Files} import play.api.Play.current @@ -50,6 +52,7 @@ case class PlaintextBallot(id: Long = -1, answers: Array[Answer] = Array[Answer] case class PlaintextError(message: String) extends Exception(message) case class DumpPksError(message: String) extends Exception(message) case class BallotEncryptionError(message: String) extends Exception(message) +case class GetElectionInfoError(message: String) extends Exception(message) object PlaintextBallot { val ID = 0 @@ -225,6 +228,44 @@ object Console { khmac } + private def get_election_info(electionId: Long) : Future[ElectionDTO] = { + val promise = Promise[ElectionDTO] + Future { + val url = s"http://$host:$port/api/election/$electionId" + wsClient.url(url) .get() map { response => + if(response.status == HTTP.ACCEPTED) { + val dto = response.json.validate[ElectionDTO].get + promise success dto + } else { + promise failure GetElectionInfoError(s"HTTP GET request to $url returned status: ${response.status} and body: ${response.body}") + } + } recover { case error: Throwable => + promise failure error + } + } recover { case error: Throwable => + promise failure error + } + promise.future + } + + private def get_election_info_all(electionsSet: scala.collection.immutable.Set[Long]) : Future[scala.collection.mutable.HashMap[Long, ElectionDTO]] = { + val promise = Promise[scala.collection.mutable.HashMap[Long, ElectionDTO]]() + Future { + val map = scala.collection.mutable.HashMap[Long, ElectionDTO]() + electionsSet.par.foreach { eid => + get_election_info(eid) map { dto : ElectionDTO => + this.synchronized { + map += (dto.id -> dto) + } + } + } + promise success ( map ) + } recover { case error: Throwable => + promise failure error + } + promise.future + } + private def dump_pks(electionId: Long): Future[Unit] = { val promise = Promise[Unit]() Future { @@ -250,77 +291,57 @@ object Console { private def dump_pks_elections(electionsSet: scala.collection.immutable.Set[Long]): Future[Unit] = { val promise = Promise[Unit]() Future { - var count : Long = 0 - var futuresCreated = Promise[Unit]() - for (electionId <- electionsSet) { - count += 1 - dump_pks(electionId) flatMap { - value => - futuresCreated.future map { value2 => - this.synchronized { - if ( 0 >= count ) { - throw DumpPksError("Logic error") - } - count -= 1 - if ( 0 == count ) { - promise success ( () ) - } - } - } - } recover { - case error: Throwable => - promise failure error - } - } - futuresCreated success ( () ) + electionsSet.par map ( dump_pks(_) ) + promise success ( () ) } recover { case error: Throwable => promise failure error } promise.future } - private def encryptBallotTask(ballotsList: scala.collection.immutable.List[PlaintextBallot], index: Long, numBallots: Long) : Future[Unit] = { - val promise = Promise[Unit]() - Future { - promise success ( () ) - } recover { case error: Throwable => - promise failure error + private def encodePlaintext(ballot: PlaintextBallot, dto: ElectionDTO): Array[Long] = { + var array = Array[Long](ballot.answers.length) + for (i <- 0 until array.length) { + val numChars = ( dto.configuration.questions(i).answers.length + 2 ).toString.length + var strValue : String = "" + val answer = ballot.answers(i) + for (j <- 0 until answer.options.length) { + val optionStrBase = ( answer.options(j) + 1 ).toString + strValue += "0" * (numChars - optionStrBase.length) + optionStrBase + } + array(i) = strValue.toLong } - promise.future + array } - private def encryptBallots(ballotsList: scala.collection.immutable.List[PlaintextBallot], electionsSet: scala.collection.immutable.Set[Long]): Future[Unit] = { + private def saveVote( electionId : Long, encryptedVote : EncryptedVote ) = { + + } + + private def encryptBallots( + ballotsList: scala.collection.immutable.List[PlaintextBallot], + electionsSet: scala.collection.immutable.Set[Long], + electionsInfoMap: scala.collection.mutable.HashMap[Long, ElectionDTO] + ) + : Future[Unit] = + { val promise = Promise[Unit]() Future { - var taskCounter : Long = 0 - var ballotCounter : Long = 0 - val tasksCreated = Promise[Unit]() - while (ballotCounter < vote_count) { - taskCounter += 1 - val numBallots = if (vote_count > batch_size + ballotCounter) { - batch_size - } else { - vote_count - ballotCounter - } - encryptBallotTask (ballotsList, ballotCounter % ballotsList.length, numBallots) flatMap { value => - tasksCreated.future map { value2 => - this.synchronized { - if ( 0 >= taskCounter ) { - throw BallotEncryptionError("Logic error") - } - taskCounter -= 1 - if ( 0 == taskCounter ) { - promise success ( () ) - } - } - } - } recover { - case error: Throwable => - promise failure error - } - ballotCounter += numBallots + val votes = ballotsList.par.map{ ballot : PlaintextBallot => + (ballot.id, encodePlaintext( ballot, electionsInfoMap.get(ballot.id).get ) ) + }.seq + val toEncrypt = { + val extraSize = vote_count - votes.length + val extra = Array.fill(extraSize.toInt){ votes(scala.util.Random.nextInt(votes.length)) } + votes ++ extra } - tasksCreated success ( () ) + toEncrypt.par.map { case (electionId, plaintext) => + val jsonPks = Json.parse(electionsInfoMap.get(electionId).get.pks.get) + val pks = jsonPks.validate[Array[PublicKey]].get + val encryptedVote = Crypto.encrypt(pks, plaintext) + saveVote(electionId, encryptedVote) + } + promise success ( () ) } recover { case error: Throwable => promise failure error } @@ -331,8 +352,11 @@ object Console { val promise = Promise[Unit]() Future { promise completeWith { parsePlaintexts() flatMap { case (ballotsList, electionsSet) => - dump_pks_elections(electionsSet) flatMap { case u => - encryptBallots(ballotsList, electionsSet) + val dumpPksFuture = dump_pks_elections(electionsSet) + get_election_info_all(electionsSet) flatMap { case electionsInfoMap => + dumpPksFuture flatMap { case pksDumped => + encryptBallots(ballotsList, electionsSet, electionsInfoMap) + } } } } From f9f80ffdafa559d8dd394fba1ec8271c482c6fcb Mon Sep 17 00:00:00 2001 From: Findeton Date: Wed, 15 Mar 2017 18:23:45 +0100 Subject: [PATCH 37/68] write csv with encrypted ballots --- app/commands/Console.scala | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 1ba43efc..15b315c7 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -36,7 +36,12 @@ import play.libs.Akka import play.api.libs.json._ -import java.nio.file.{Paths, Files} +import java.nio.file.{Paths, Files, Path} +import java.nio.file.StandardOpenOption._ +import java.nio.charset.StandardCharsets +import java.io.File +import java.io.RandomAccessFile +import java.nio._ import play.api.Play.current import play.api.libs.ws._ @@ -59,6 +64,16 @@ object PlaintextBallot { val ANSWER = 1 } + +class VotesWriter(filePath: Path) { + Files.write(filePath, "".getBytes(StandardCharsets.UTF_8), CREATE, TRUNCATE_EXISTING) + def write(id: Long, ballot: EncryptedVote) = Future { + val content: String = id.toString + "-" + Json.toJson(ballot).toString + "\n" + this.synchronized { + Files.write(filePath, content.getBytes(StandardCharsets.UTF_8), APPEND) + } + } +} /** * Command for admin purposes * @@ -78,7 +93,6 @@ object Console { var ciphertexts_path = "ciphertexts.csv" var shared_secret = "" var datastore = "/home/agoraelections/datastore" - var batch_size : Long = 50 // copied from https://www.playframework.com/documentation/2.3.x/ScalaWS val clientConfig = new DefaultWSClientConfig() @@ -106,9 +120,6 @@ object Console { } else if ("--shared-secret" == args(arg_index + 1)) { shared_secret = args(arg_index + 2) arg_index += 2 - } else if ("--batch-size" == args(arg_index + 1)) { - batch_size = args(arg_index + 2).toLong - arg_index += 2 } else if ("--port" == args(arg_index + 1)) { port = args(arg_index + 2).toLong if (port <= 0 || port > 65535) { @@ -314,10 +325,6 @@ object Console { array } - private def saveVote( electionId : Long, encryptedVote : EncryptedVote ) = { - - } - private def encryptBallots( ballotsList: scala.collection.immutable.List[PlaintextBallot], electionsSet: scala.collection.immutable.Set[Long], @@ -335,11 +342,12 @@ object Console { val extra = Array.fill(extraSize.toInt){ votes(scala.util.Random.nextInt(votes.length)) } votes ++ extra } + val writer = new VotesWriter(Paths.get(ciphertexts_path)) toEncrypt.par.map { case (electionId, plaintext) => val jsonPks = Json.parse(electionsInfoMap.get(electionId).get.pks.get) val pks = jsonPks.validate[Array[PublicKey]].get val encryptedVote = Crypto.encrypt(pks, plaintext) - saveVote(electionId, encryptedVote) + writer.write(electionId, encryptedVote) } promise success ( () ) } recover { case error: Throwable => From ff769774d4964829e82f9f548a0417c2b4924cb9 Mon Sep 17 00:00:00 2001 From: Findeton Date: Wed, 15 Mar 2017 18:25:52 +0100 Subject: [PATCH 38/68] improve writing file --- app/commands/Console.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 15b315c7..5b4b2476 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -65,12 +65,13 @@ object PlaintextBallot { } -class VotesWriter(filePath: Path) { - Files.write(filePath, "".getBytes(StandardCharsets.UTF_8), CREATE, TRUNCATE_EXISTING) +class VotesWriter(path: Path) { + Files.deleteIfExists(path) + Files.createFile(path) def write(id: Long, ballot: EncryptedVote) = Future { val content: String = id.toString + "-" + Json.toJson(ballot).toString + "\n" this.synchronized { - Files.write(filePath, content.getBytes(StandardCharsets.UTF_8), APPEND) + Files.write(path, content.getBytes(StandardCharsets.UTF_8), APPEND) } } } From a26ea9ca4fad33a3181a038f1fb9c5150a1ca016 Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 16 Mar 2017 11:57:25 +0100 Subject: [PATCH 39/68] fixing futures --- app/commands/Console.scala | 57 +++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 5b4b2476..d24ec730 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -264,14 +264,18 @@ object Console { val promise = Promise[scala.collection.mutable.HashMap[Long, ElectionDTO]]() Future { val map = scala.collection.mutable.HashMap[Long, ElectionDTO]() - electionsSet.par.foreach { eid => - get_election_info(eid) map { dto : ElectionDTO => - this.synchronized { - map += (dto.id -> dto) + promise completeWith { + Future.traverse(electionsSet) { eid => + get_election_info(eid) map { dto : ElectionDTO => + this.synchronized { + map += (dto.id -> dto) + } + () } + } map { _ => + map } } - promise success ( map ) } recover { case error: Throwable => promise failure error } @@ -303,8 +307,11 @@ object Console { private def dump_pks_elections(electionsSet: scala.collection.immutable.Set[Long]): Future[Unit] = { val promise = Promise[Unit]() Future { - electionsSet.par map ( dump_pks(_) ) - promise success ( () ) + promise completeWith { + Future.traverse( electionsSet ) ( x => dump_pks(x) ) map { _ => + () + } + } } recover { case error: Throwable => promise failure error } @@ -338,19 +345,25 @@ object Console { val votes = ballotsList.par.map{ ballot : PlaintextBallot => (ballot.id, encodePlaintext( ballot, electionsInfoMap.get(ballot.id).get ) ) }.seq - val toEncrypt = { + val toEncrypt : Seq[(Long, Array[Long])] = { val extraSize = vote_count - votes.length val extra = Array.fill(extraSize.toInt){ votes(scala.util.Random.nextInt(votes.length)) } votes ++ extra } val writer = new VotesWriter(Paths.get(ciphertexts_path)) - toEncrypt.par.map { case (electionId, plaintext) => - val jsonPks = Json.parse(electionsInfoMap.get(electionId).get.pks.get) - val pks = jsonPks.validate[Array[PublicKey]].get - val encryptedVote = Crypto.encrypt(pks, plaintext) - writer.write(electionId, encryptedVote) + promise completeWith { + Future.traverse (toEncrypt) { case (electionId, plaintext) => + Future { + val jsonPks = Json.parse(electionsInfoMap.get(electionId).get.pks.get) + val pks = jsonPks.validate[Array[PublicKey]].get + val encryptedVote = Crypto.encrypt(pks, plaintext) + writer.write(electionId, encryptedVote) + () + } + } map { _ => + () + } } - promise success ( () ) } recover { case error: Throwable => promise failure error } @@ -360,13 +373,17 @@ object Console { private def gen_votes(): Future[Unit] = { val promise = Promise[Unit]() Future { - promise completeWith { parsePlaintexts() flatMap { case (ballotsList, electionsSet) => - val dumpPksFuture = dump_pks_elections(electionsSet) - get_election_info_all(electionsSet) flatMap { case electionsInfoMap => - dumpPksFuture flatMap { case pksDumped => - encryptBallots(ballotsList, electionsSet, electionsInfoMap) + promise completeWith { + parsePlaintexts() flatMap { + case (ballotsList, electionsSet) => + val dumpPksFuture = dump_pks_elections(electionsSet) + get_election_info_all(electionsSet) flatMap { + case electionsInfoMap => + dumpPksFuture flatMap { + case pksDumped => + encryptBallots(ballotsList, electionsSet, electionsInfoMap) + } } - } } } } recover { case error: Throwable => From 735646bc9c5067d3384b9cce2a00f9a7464a9453 Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 16 Mar 2017 14:07:57 +0100 Subject: [PATCH 40/68] fixing errors --- app/commands/Console.scala | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index d24ec730..d914c3ef 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -59,6 +59,7 @@ case class DumpPksError(message: String) extends Exception(message) case class BallotEncryptionError(message: String) extends Exception(message) case class GetElectionInfoError(message: String) extends Exception(message) + object PlaintextBallot { val ID = 0 val ANSWER = 1 @@ -92,7 +93,7 @@ object Console { var port : Long = 9000 var plaintexts_path = "plaintexts.txt" var ciphertexts_path = "ciphertexts.csv" - var shared_secret = "" + var shared_secret = "" var datastore = "/home/agoraelections/datastore" // copied from https://www.playframework.com/documentation/2.3.x/ScalaWS @@ -198,6 +199,9 @@ object Console { // add the last Answer optionsBuffer match { case Some(optionsBufferValue) => + if (strIndex.isDefined) { + optionsBufferValue += strIndex.get.toLong + } answersBuffer += Answer(optionsBufferValue.toArray) case None => throw PlaintextError(s"Error on line $lineNumber: unknown error, invalid state. Line: $line") @@ -245,8 +249,8 @@ object Console { Future { val url = s"http://$host:$port/api/election/$electionId" wsClient.url(url) .get() map { response => - if(response.status == HTTP.ACCEPTED) { - val dto = response.json.validate[ElectionDTO].get + if(response.status == HTTP.OK) { + val dto = (response.json \ "payload").validate[ElectionDTO].get promise success dto } else { promise failure GetElectionInfoError(s"HTTP GET request to $url returned status: ${response.status} and body: ${response.body}") @@ -290,7 +294,7 @@ object Console { wsClient.url(url) .withHeaders("Authorization" -> auth) .post(Results.EmptyContent()).map { response => - if(response.status == HTTP.ACCEPTED) { + if(response.status == HTTP.OK) { promise success ( () ) } else { promise failure DumpPksError(s"HTTP POST request to $url returned status: ${response.status} and body: ${response.body}") @@ -319,7 +323,7 @@ object Console { } private def encodePlaintext(ballot: PlaintextBallot, dto: ElectionDTO): Array[Long] = { - var array = Array[Long](ballot.answers.length) + var array = new Array[Long](ballot.answers.length) for (i <- 0 until array.length) { val numChars = ( dto.configuration.questions(i).answers.length + 2 ).toString.length var strValue : String = "" @@ -328,6 +332,11 @@ object Console { val optionStrBase = ( answer.options(j) + 1 ).toString strValue += "0" * (numChars - optionStrBase.length) + optionStrBase } + // blank vote + if (0 == answer.options.length) { + val optionStrBase = ( dto.configuration.questions(i).answers.length + 2 ).toString + strValue += "0" * (numChars - optionStrBase.length) + optionStrBase + } array(i) = strValue.toLong } array @@ -378,9 +387,9 @@ object Console { case (ballotsList, electionsSet) => val dumpPksFuture = dump_pks_elections(electionsSet) get_election_info_all(electionsSet) flatMap { - case electionsInfoMap => + electionsInfoMap => dumpPksFuture flatMap { - case pksDumped => + pksDumped => encryptBallots(ballotsList, electionsSet, electionsInfoMap) } } From f1d524275a87012d86484eb7a551a700ef075585 Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 16 Mar 2017 15:51:07 +0100 Subject: [PATCH 41/68] fix write futures --- app/commands/Console.scala | 10 ++++++---- app/what.csv | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 app/what.csv diff --git a/app/commands/Console.scala b/app/commands/Console.scala index d914c3ef..9cfaa95d 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -69,7 +69,7 @@ object PlaintextBallot { class VotesWriter(path: Path) { Files.deleteIfExists(path) Files.createFile(path) - def write(id: Long, ballot: EncryptedVote) = Future { + def write(id: Long, ballot: EncryptedVote) : Future[Unit] = Future { val content: String = id.toString + "-" + Json.toJson(ballot).toString + "\n" this.synchronized { Files.write(path, content.getBytes(StandardCharsets.UTF_8), APPEND) @@ -274,7 +274,6 @@ object Console { this.synchronized { map += (dto.id -> dto) } - () } } map { _ => map @@ -362,13 +361,16 @@ object Console { val writer = new VotesWriter(Paths.get(ciphertexts_path)) promise completeWith { Future.traverse (toEncrypt) { case (electionId, plaintext) => + val writePromise = Promise[Unit]() Future { val jsonPks = Json.parse(electionsInfoMap.get(electionId).get.pks.get) val pks = jsonPks.validate[Array[PublicKey]].get val encryptedVote = Crypto.encrypt(pks, plaintext) - writer.write(electionId, encryptedVote) - () + writePromise completeWith writer.write(electionId, encryptedVote) + } recover { case error: Throwable => + writePromise failure error } + writePromise.future } map { _ => () } diff --git a/app/what.csv b/app/what.csv new file mode 100644 index 00000000..ae14f4b8 --- /dev/null +++ b/app/what.csv @@ -0,0 +1,2 @@ +hello world file +dd \ No newline at end of file From 7692acb950e80bd2dfaf8203bd03fd252cc80894 Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 16 Mar 2017 18:26:13 +0100 Subject: [PATCH 42/68] add_kmacs --- admin/admin.py | 2 - admin/plaintexts.txt | 2 + app/commands/Console.scala | 79 ++++++++++++++++++++++++++++++++------ 3 files changed, 70 insertions(+), 13 deletions(-) create mode 100644 admin/plaintexts.txt diff --git a/admin/admin.py b/admin/admin.py index 9a18b3d4..aca7e5e1 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -833,13 +833,11 @@ def gen_all_khmacs(vote_count, election_id): import hmac start_time = time.time() alphabet = '0123456789abcdef' - timestamp = 1000 * int(time.time()) counter = 0 khmac_list = [] voterid_len = 28 while counter < vote_count: voterid = gen_rnd_str(voterid_len, alphabet) - message = '%s:AuthEvent:%i:vote:%s' % (voterid, election_id, timestamp) khmac = get_hmac(cfg, voterid, "AuthEvent", election_id, 'vote') khmac_list.append((voterid, khmac)) counter += 1 diff --git a/admin/plaintexts.txt b/admin/plaintexts.txt new file mode 100644 index 00000000..d5bc45cb --- /dev/null +++ b/admin/plaintexts.txt @@ -0,0 +1,2 @@ +22|56,2,39||5 +39|0|1 \ No newline at end of file diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 9cfaa95d..4e5f3101 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -69,8 +69,8 @@ object PlaintextBallot { class VotesWriter(path: Path) { Files.deleteIfExists(path) Files.createFile(path) - def write(id: Long, ballot: EncryptedVote) : Future[Unit] = Future { - val content: String = id.toString + "-" + Json.toJson(ballot).toString + "\n" + + def write(content: String) : Future[Unit] = Future { this.synchronized { Files.write(path, content.getBytes(StandardCharsets.UTF_8), APPEND) } @@ -93,8 +93,11 @@ object Console { var port : Long = 9000 var plaintexts_path = "plaintexts.txt" var ciphertexts_path = "ciphertexts.csv" + var ciphertexts_khmac_path = "ciphertexts-khmac.csv" var shared_secret = "" var datastore = "/home/agoraelections/datastore" + var voterid_len : Int = 28 + val voterid_alphabet: String = "0123456789abcdef" // copied from https://www.playframework.com/documentation/2.3.x/ScalaWS val clientConfig = new DefaultWSClientConfig() @@ -105,6 +108,7 @@ object Console { implicit val wsClient = new play.api.libs.ws.ning.NingWSClient(secureDefaultsWithSpecificOptions) + private def parse_args(args: Array[String]) = { var arg_index = 0 while (arg_index + 2 < args.length) { @@ -119,6 +123,12 @@ object Console { } else if ("--ciphertexts" == args(arg_index + 1)) { ciphertexts_path = args(arg_index + 2) arg_index += 2 + } else if ("--ciphertexts-khmac" == args(arg_index + 1)) { + ciphertexts_khmac_path = args(arg_index + 2) + arg_index += 2 + } else if ("--voterid-len" == args(arg_index + 1)) { + voterid_len = args(arg_index + 2).toInt + arg_index += 2 } else if ("--shared-secret" == args(arg_index + 1)) { shared_secret = args(arg_index + 2) arg_index += 2 @@ -210,6 +220,32 @@ object Console { ballot } + private def add_khmacs() : Future[Unit] = { + val promise = Promise[Unit]() + Future { + if (Files.exists(Paths.get(ciphertexts_path))) { + val now = Some(System.currentTimeMillis / 1000) + val writer = new VotesWriter(Paths.get(ciphertexts_khmac_path)) + promise completeWith { + Future.traverse (io.Source.fromFile(ciphertexts_path).getLines()) { file_line => + val array = file_line.split('-') + val eid = array(0).toLong + val voterId = array(1) + val khmac = get_khmac(voterId, "AuthEvent", eid, "vote", now) + writer.write(file_line + "-" + khmac) + } map { _ => + () + } + } + } else { + throw new java.io.FileNotFoundException("tally does not exist") + } + } recover { case error: Throwable => + promise failure error + } + promise.future + } + private def parsePlaintexts(): Future[(scala.collection.immutable.List[PlaintextBallot], scala.collection.immutable.Set[Long])] = { val promise = Promise[(scala.collection.immutable.List[PlaintextBallot], scala.collection.immutable.Set[Long])]() Future { @@ -236,8 +272,11 @@ object Console { promise.future } - private def get_khmac(userId: String, objType: String, objId: Long, perm: String) : String = { - val now: Long = System.currentTimeMillis / 1000 + private def get_khmac(userId: String, objType: String, objId: Long, perm: String, nowOpt: Option[Long] = None) : String = { + val now: Long = nowOpt match { + case Some(time) => time + case None => System.currentTimeMillis / 1000 + } val message = s"$userId:$objType:$objId:$perm:$now" val hmac = Crypto.hmac(shared_secret, message) val khmac = s"khmac:///sha-256;$hmac/$message" @@ -341,6 +380,25 @@ object Console { array } + private def gen_rnd_str(len: Int, choices: String) : String = { + (0 until len) map { _ => + choices( scala.util.Random.nextInt(choices.length) ) + } mkString + } + + private def generate_voterid() : String = { + gen_rnd_str(voterid_len, voterid_alphabet) + } + + private def generate_vote_line(id: Long, ballot: EncryptedVote) : String = { + val vote = Json.toJson(ballot).toString + val voteHash = Crypto.sha256(vote) + val voterId = generate_voterid() + val voteDTO = Json.toJson(VoteDTO(vote, voteHash)).toString + val line = id.toString + "-" + generate_voterid() + "-" + voteDTO + "\n" + line + } + private def encryptBallots( ballotsList: scala.collection.immutable.List[PlaintextBallot], electionsSet: scala.collection.immutable.Set[Long], @@ -366,7 +424,8 @@ object Console { val jsonPks = Json.parse(electionsInfoMap.get(electionId).get.pks.get) val pks = jsonPks.validate[Array[PublicKey]].get val encryptedVote = Crypto.encrypt(pks, plaintext) - writePromise completeWith writer.write(electionId, encryptedVote) + val line = generate_vote_line(electionId, encryptedVote) + writePromise completeWith writer.write(line) } recover { case error: Throwable => writePromise failure error } @@ -403,10 +462,7 @@ object Console { promise.future } - private def send_votes() = { - } - - def main(args: Array[String]) = { + def main(args: Array[String]) : Unit = { if(0 == args.length) { showHelp() } else { @@ -421,11 +477,12 @@ object Console { println("gen_votes error " + error) System.exit(-1) } - } else if ( "send_votes" == command) { - send_votes() + } else if ( "add_khmacs" == command) { + add_khmacs() } else { showHelp() } } + () } } \ No newline at end of file From 0b77a005b252029be33a748a2d8ebdcecaaedef1 Mon Sep 17 00:00:00 2001 From: Findeton Date: Fri, 17 Mar 2017 14:34:57 +0100 Subject: [PATCH 43/68] add comments, fix some issues --- app/commands/Console.scala | 273 +++++++++++++++++++++++++++------ app/models/Models.scala | 4 + app/utils/JsonFormatters.scala | 8 + 3 files changed, 242 insertions(+), 43 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 4e5f3101..13985837 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -51,22 +51,26 @@ import play.api.mvc._ import play.api.http.{Status => HTTP} -case class Answer(options: Array[Long] = Array[Long]()) -// id is the election ID -case class PlaintextBallot(id: Long = -1, answers: Array[Answer] = Array[Answer]()) case class PlaintextError(message: String) extends Exception(message) case class DumpPksError(message: String) extends Exception(message) case class BallotEncryptionError(message: String) extends Exception(message) case class GetElectionInfoError(message: String) extends Exception(message) +case class EncodePlaintextError(message: String) extends Exception(message) +case class EncryptionError(message: String) extends Exception(message) - +/** + * This object contains the states required for reading a plaintext ballot + * It's used on Console.processPlaintextLine + */ object PlaintextBallot { - val ID = 0 - val ANSWER = 1 + val ID = 0 // reading election ID + val ANSWER = 1 // reading answers } - -class VotesWriter(path: Path) { +/** + * A simple class to write lines to a single file, in a multi-threading safe way + */ +class FileWriter(path: Path) { Files.deleteIfExists(path) Files.createFile(path) @@ -88,17 +92,25 @@ object Console { //implicit val ec = ExecutionContext.fromExecutor(new ForkJoinPool(100)) //implicit val slickExecutionContext = Akka.system.dispatchers.lookup("play.akka.actor.slick-context") + // number of votes to create var vote_count : Long = 0 + // hostname of the agora-elections server var host = "localhost" + // agora-elections port var port : Long = 9000 + // default plaintext path var plaintexts_path = "plaintexts.txt" + // default ciphertexts path var ciphertexts_path = "ciphertexts.csv" + // default ciphertexts (with khmac) path var ciphertexts_khmac_path = "ciphertexts-khmac.csv" var shared_secret = "" - var datastore = "/home/agoraelections/datastore" + // default voter id length (number of characters) var voterid_len : Int = 28 val voterid_alphabet: String = "0123456789abcdef" + // In order to make http requests with Play without a running Play instance, + // we have to do this // copied from https://www.playframework.com/documentation/2.3.x/ScalaWS val clientConfig = new DefaultWSClientConfig() val secureDefaults:com.ning.http.client.AsyncHttpClientConfig = new NingAsyncHttpClientConfigBuilder(clientConfig).build() @@ -108,7 +120,9 @@ object Console { implicit val wsClient = new play.api.libs.ws.ning.NingWSClient(secureDefaultsWithSpecificOptions) - + /** + * Parse program arguments + */ private def parse_args(args: Array[String]) = { var arg_index = 0 while (arg_index + 2 < args.length) { @@ -142,19 +156,66 @@ object Console { throw new java.lang.IllegalArgumentException("Unrecognized argument: " + args(arg_index + 1)) } } - } + } + /** + * Prints to the standard output how to use this program + */ private def showHelp() = { - System.out.println("showHelp") + System.out.println( + """NAME + | commands.Console - Generate and send votes for benchmark purposes + | + |SYNOPSIS + | commands.Console [gen_votes|add_khmacs] [--vote-count number] + | [--plaintexts path] [--ciphertexts path] [--ciphertexts-khmac path] + | [--voterid-len number] [--shared-secret password] [--port port] + | + |DESCRIPTION + | In order to run this program, use: + | activator "runMain commands.Console [arguments]" + | possible commands: + | - gen_votes + | - add_khmacs + | Possible arguments + | - vote-count + | - plaintexts + | - ciphertexts + | - ciphertexts-khmac + | - voterid-len + | - shared-secret + | - port + |""".stripMargin) } + /** + * Processes a plaintext ballot line + * A plaintext ballot will have the following format: + * id|option1,option2|option1|| + * + * - The id is a number that corresponds to the election id + * - A separator '|' is used to signal the end of the election id + * - The chosen options for each answer come after the election id + * - The options for an answer are separated by a ',' + * - Each answer is separated by a '|' + * - A blank vote for an answer is represented with no characters + * - For example '||' means a blank vote for the corresponding answer + */ private def processPlaintextLine(line: String, lineNumber: Long) : PlaintextBallot = { + // in this variable we keep adding the characters that will form a complete number var strIndex: Option[String] = None + // state: either reading the election index or the election answers var state = PlaintextBallot.ID - var ballot = PlaintextBallot() + // the ballot to be returned + var ballot = new PlaintextBallot() + // for each question in the election, there will be zero (blank vote) to n possible chosen options + // We'll keep here the options chosen for each answer as we are reading them var optionsBuffer: Option[ArrayBuffer[Long]] = None - var answersBuffer: ArrayBuffer[Answer] = ArrayBuffer[Answer]() - for (i <- 0 until line.length) { + // buffer that holds the chosen options for the answers + var answersBuffer: ArrayBuffer[PlaintextAnswer] = ArrayBuffer[PlaintextAnswer]() + + // iterate through all characters in the string line + for (i <- 0 until line.length) { val c = line.charAt(i) if(c.isDigit) { // keep reading digits till we get the whole number strIndex match { @@ -165,31 +226,39 @@ object Console { // add the first character to the string containing a number strIndex = Some(c.toString) } - } else { + } + // it's not a digit + else { + // state: reading election ID if (PlaintextBallot.ID == state) { if ('|' != c) { throw PlaintextError(s"Error on line $lineNumber, character $i: character separator '|' not found after election index . Line: $line") } strIndex match { case Some(strIndexValue) => - ballot = PlaintextBallot(strIndex.get.toLong, ballot.answers) + ballot = new PlaintextBallot(strIndex.get.toLong, ballot.answers) strIndex = None optionsBuffer = Some(ArrayBuffer[Long]()) state = PlaintextBallot.ANSWER case None => throw PlaintextError(s"Error on line $lineNumber, character $i: election index not recognized. Line: $line") } - } else if (PlaintextBallot.ANSWER == state) { + } + // state: reading answers to each question + else if (PlaintextBallot.ANSWER == state) { optionsBuffer match { case Some(optionsBufferValue) => + // end of this question, add question to buffer if ('|' == c) { if (strIndex.isDefined) { optionsBufferValue += strIndex.get.toLong strIndex = None } - answersBuffer += Answer(optionsBufferValue.toArray) + answersBuffer += PlaintextAnswer(optionsBufferValue.toArray) optionsBuffer = Some(ArrayBuffer[Long]()) - } else if(',' == c) { + } + // add chosen option to buffer + else if(',' == c) { strIndex match { case Some(strIndexValue) => optionsBufferValue += strIndexValue.toLong @@ -206,26 +275,54 @@ object Console { } } } - // add the last Answer + // add the last answer optionsBuffer match { case Some(optionsBufferValue) => + // read last option of last answer, if there's any if (strIndex.isDefined) { optionsBufferValue += strIndex.get.toLong } - answersBuffer += Answer(optionsBufferValue.toArray) + answersBuffer += PlaintextAnswer(optionsBufferValue.toArray) case None => throw PlaintextError(s"Error on line $lineNumber: unknown error, invalid state. Line: $line") } - ballot = PlaintextBallot(ballot.id, answersBuffer.toArray) + ballot = new PlaintextBallot(ballot.id, answersBuffer.toArray) ballot } + /** + * Reads a file created by gen_votes and adds a khmac to each ballot + * + * The objective of this command is to prepare the ballots to be sent by + * jmeter. + * + * Encrypting the plaintext ballots takes some time, and it can be + * done at any moment. However, the khmacs will only be valid for a certain + * period of time, and therefore, it can be useful to first encrypt the + * ballots and only generate the khmacs just before sending the ballots to + * the server. + * + * The format of each line of the ciphertexts file that this command reads + * should be: + * + * electionId-voterId-ballot + * + * Notice that the separator character is '-' + * + * The format of each line of the ciphertexts_khmac file that this command + * creates is: + * + * electionId-voterId-ballot-khmac + * + * Notice that it simply adds the khmac. + */ private def add_khmacs() : Future[Unit] = { val promise = Promise[Unit]() Future { + // check that the file we want to read exists if (Files.exists(Paths.get(ciphertexts_path))) { val now = Some(System.currentTimeMillis / 1000) - val writer = new VotesWriter(Paths.get(ciphertexts_khmac_path)) + val writer = new FileWriter(Paths.get(ciphertexts_khmac_path)) promise completeWith { Future.traverse (io.Source.fromFile(ciphertexts_path).getLines()) { file_line => val array = file_line.split('-') @@ -246,14 +343,25 @@ object Console { promise.future } + /** + * Opens the plaintexts file, and reads and parses each line. + * See Console.processPlaintextLine() comments for more info on the format + * of the plaintext file. + */ private def parsePlaintexts(): Future[(scala.collection.immutable.List[PlaintextBallot], scala.collection.immutable.Set[Long])] = { val promise = Promise[(scala.collection.immutable.List[PlaintextBallot], scala.collection.immutable.Set[Long])]() Future { - if (Files.exists(Paths.get(plaintexts_path))) { + val path = Paths.get(plaintexts_path) + // Check that the plaintexts file exists + if (Files.exists(path)) { + // buffer with all the parsed ballots val ballotsList = scala.collection.mutable.ListBuffer[PlaintextBallot]() + // set of all the election ids val electionsSet = scala.collection.mutable.LinkedHashSet[Long]() + // read all lines io.Source.fromFile(plaintexts_path).getLines().zipWithIndex.foreach { case (line, number) => + // parse line val ballot = processPlaintextLine(line, number) ballotsList += ballot electionsSet += ballot.id @@ -264,7 +372,7 @@ object Console { promise success ( ( ballotsList.sortBy(_.id).toList, electionsSet.toSet ) ) } } else { - throw new java.io.FileNotFoundException("tally does not exist") + throw new java.io.FileNotFoundException(s"plaintext file ${path.toAbsolutePath.toString} does not exist or can't be opened") } } recover { case error: Throwable => promise failure error @@ -272,6 +380,9 @@ object Console { promise.future } + /** + * Generate khmac + */ private def get_khmac(userId: String, objType: String, objId: Long, perm: String, nowOpt: Option[Long] = None) : String = { val now: Long = nowOpt match { case Some(time) => time @@ -283,6 +394,10 @@ object Console { khmac } + /** + * Makes an http request to agora-elections to get the election info. + * Returns the parsed election info + */ private def get_election_info(electionId: Long) : Future[ElectionDTO] = { val promise = Promise[ElectionDTO] Future { @@ -303,6 +418,10 @@ object Console { promise.future } + /** + * Given a set of election ids, it returns a map of the election ids and their + * election info. + */ private def get_election_info_all(electionsSet: scala.collection.immutable.Set[Long]) : Future[scala.collection.mutable.HashMap[Long, ElectionDTO]] = { val promise = Promise[scala.collection.mutable.HashMap[Long, ElectionDTO]]() Future { @@ -310,6 +429,7 @@ object Console { promise completeWith { Future.traverse(electionsSet) { eid => get_election_info(eid) map { dto : ElectionDTO => + // using synchronized to make it thread-safe this.synchronized { map += (dto.id -> dto) } @@ -324,6 +444,10 @@ object Console { promise.future } + /** + * Makes an HTTP request to agora-elections to dump the public keys for a + * given election id. + */ private def dump_pks(electionId: Long): Future[Unit] = { val promise = Promise[Unit]() Future { @@ -346,6 +470,10 @@ object Console { promise.future } + /** + * Given a set of election ids, it returns a future that will be completed + * when all the public keys of all those elections have been dumped. + */ private def dump_pks_elections(electionsSet: scala.collection.immutable.Set[Long]): Future[Unit] = { val promise = Promise[Unit]() Future { @@ -360,36 +488,75 @@ object Console { promise.future } + /** + * Given a plaintext and the election info, it encodes each question's answers + * into a number, ready to be encrypted + */ private def encodePlaintext(ballot: PlaintextBallot, dto: ElectionDTO): Array[Long] = { var array = new Array[Long](ballot.answers.length) + if (dto.configuration.questions.length != ballot.answers.length) { + val strBallot = Json.toJson(ballot).toString + val strDto = Json.toJson(dto).toString + throw EncodePlaintextError( + s"""Plaintext ballot:\n$strBallot\nElection dto:\n$strDto\n + |Error: wrong number of questions on the plaintext ballot: + |${dto.configuration.questions.length} != ${ballot.answers.length}""") + } for (i <- 0 until array.length) { - val numChars = ( dto.configuration.questions(i).answers.length + 2 ).toString.length - var strValue : String = "" - val answer = ballot.answers(i) - for (j <- 0 until answer.options.length) { - val optionStrBase = ( answer.options(j) + 1 ).toString - strValue += "0" * (numChars - optionStrBase.length) + optionStrBase - } - // blank vote - if (0 == answer.options.length) { - val optionStrBase = ( dto.configuration.questions(i).answers.length + 2 ).toString - strValue += "0" * (numChars - optionStrBase.length) + optionStrBase + Try { + val numChars = ( dto.configuration.questions(i).answers.length + 2 ).toString.length + // holds the value of the encoded answer, before converting it to a Long + var strValue : String = "" + val answer = ballot.answers(i) + for (j <- 0 until answer.options.length) { + // sum 1 as the encryption method can't encode value zero + val optionStrBase = ( answer.options(j) + 1 ).toString + // Each chosen option needs to have the same length in number of + // characters/digits, so we fill in with zeros on the left + strValue += "0" * (numChars - optionStrBase.length) + optionStrBase + } + // blank vote + if (0 == answer.options.length) { + val optionStrBase = ( dto.configuration.questions(i).answers.length + 2 ).toString + strValue += "0" * (numChars - optionStrBase.length) + optionStrBase + } + // Convert to long. Notice that the zeros on the left added by the last + // chosen option won't be included + array(i) = strValue.toLong + } match { + case Failure(error) => + val strBallot = Json.toJson(ballot).toString + val strDto = Json.toJson(dto).toString + throw EncodePlaintextError(s"Plaintext ballot:\n$strBallot\nElection dto:\n$strDto\nQuestion index where error was generated: $i\nError: ${error.getMessage}") + case _ => } - array(i) = strValue.toLong } array } + /** + * Generate a random string with a given length and alphabet + * Note: This is not truly random, it uses a pseudo-random generator + */ private def gen_rnd_str(len: Int, choices: String) : String = { (0 until len) map { _ => choices( scala.util.Random.nextInt(choices.length) ) } mkString } + /** + * Generate a random voter id + */ private def generate_voterid() : String = { gen_rnd_str(voterid_len, voterid_alphabet) } + /** + * Given an election id and an encrypted ballot, it generates a String text + * line in the following format: + * electionId - voterId - vote + * Notice that it automatically adds a random voter id + */ private def generate_vote_line(id: Long, ballot: EncryptedVote) : String = { val vote = Json.toJson(ballot).toString val voteHash = Crypto.sha256(vote) @@ -399,30 +566,47 @@ object Console { line } + /** + * Given a list of plaintext ballots and a map of election ids and their + * election info, it generates and encrypts the ballots, saving them to a + * file. + * Read generate_vote_line for more info on the output file format. + */ private def encryptBallots( ballotsList: scala.collection.immutable.List[PlaintextBallot], - electionsSet: scala.collection.immutable.Set[Long], electionsInfoMap: scala.collection.mutable.HashMap[Long, ElectionDTO] ) : Future[Unit] = { val promise = Promise[Unit]() Future { + // base list of encoded plaintext ballots val votes = ballotsList.par.map{ ballot : PlaintextBallot => (ballot.id, encodePlaintext( ballot, electionsInfoMap.get(ballot.id).get ) ) }.seq + // map election ids and public keys + val pksMap = scala.collection.mutable.HashMap[Long, Array[PublicKey]]() + electionsInfoMap foreach { case (key, value) => + val jsonPks = Json.toJson(value.pks.get) + val pks = jsonPks.validate[Array[PublicKey]].get + pksMap += (key -> pks) + } + // we need to generate vote_count encrypted ballots, fill the list with + // random samples of the base list val toEncrypt : Seq[(Long, Array[Long])] = { val extraSize = vote_count - votes.length val extra = Array.fill(extraSize.toInt){ votes(scala.util.Random.nextInt(votes.length)) } votes ++ extra } - val writer = new VotesWriter(Paths.get(ciphertexts_path)) + val writer = new FileWriter(Paths.get(ciphertexts_path)) promise completeWith { Future.traverse (toEncrypt) { case (electionId, plaintext) => val writePromise = Promise[Unit]() Future { - val jsonPks = Json.parse(electionsInfoMap.get(electionId).get.pks.get) - val pks = jsonPks.validate[Array[PublicKey]].get + val pks = pksMap.get(electionId).get + if (pks.length != plaintext.length) { + throw new EncryptionError(s"${pks.length} != ${plaintext.length}") + } val encryptedVote = Crypto.encrypt(pks, plaintext) val line = generate_vote_line(electionId, encryptedVote) writePromise completeWith writer.write(line) @@ -440,6 +624,9 @@ object Console { promise.future } + /** + * Given a list of plaintexts, it generates their ciphertexts + */ private def gen_votes(): Future[Unit] = { val promise = Promise[Unit]() Future { @@ -451,7 +638,7 @@ object Console { electionsInfoMap => dumpPksFuture flatMap { pksDumped => - encryptBallots(ballotsList, electionsSet, electionsInfoMap) + encryptBallots(ballotsList, electionsInfoMap) } } } diff --git a/app/models/Models.scala b/app/models/Models.scala index e1d38cd3..dd242746 100644 --- a/app/models/Models.scala +++ b/app/models/Models.scala @@ -645,3 +645,7 @@ case class Popk(challenge: BigInt, commitment: BigInt, response: BigInt) /** data describing an authority, used in admin interface */ case class AuthData(name: Option[String], description: Option[String], url: Option[String], image: Option[String]) + +case class PlaintextAnswer(options: Array[Long] = Array[Long]()) +// id is the election ID +case class PlaintextBallot(id: Long = -1, answers: Array[PlaintextAnswer] = Array[PlaintextAnswer]()) diff --git a/app/utils/JsonFormatters.scala b/app/utils/JsonFormatters.scala index d4b952f1..d82367dc 100644 --- a/app/utils/JsonFormatters.scala +++ b/app/utils/JsonFormatters.scala @@ -85,4 +85,12 @@ object JsonFormatters { implicit val tallyResponseF = Json.format[TallyResponse] implicit val authDataF = Json.format[AuthData] + + implicit val plaintextAnswerW : Writes[PlaintextAnswer] = + (JsPath \ "options").write[Array[Long]] contramap { (t: PlaintextAnswer) => t.options } + + implicit val plaintextAnswerR : Reads[PlaintextAnswer] = + (JsPath \ "options").read[Array[Long]] map (PlaintextAnswer.apply ) + + implicit val PlaintextBallotF = Json.format[PlaintextBallot] } From d6192fe82a2a9fb060a93cfdead069a8867c3adf Mon Sep 17 00:00:00 2001 From: Findeton Date: Fri, 17 Mar 2017 16:03:46 +0100 Subject: [PATCH 44/68] fix json parse of pks --- app/commands/Console.scala | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 13985837..d13beeb3 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -587,9 +587,28 @@ object Console { // map election ids and public keys val pksMap = scala.collection.mutable.HashMap[Long, Array[PublicKey]]() electionsInfoMap foreach { case (key, value) => - val jsonPks = Json.toJson(value.pks.get) - val pks = jsonPks.validate[Array[PublicKey]].get - pksMap += (key -> pks) + val strPks = value.pks match { + case Some(strPks) => + strPks + case None => + val strDto = Json.toJson(value).toString + throw new GetElectionInfoError(s"Error: no public keys found for election $key. Election Info: $strDto") + } + val jsonPks = Try { + Json.parse(strPks) + } match { + case Success(jsonPks) => + jsonPks + case Failure(error) => + throw new GetElectionInfoError(s"Error: public keys with invalid JSON format for election $key.\nPublic keys: $strPks.\n${error.getMessage}") + } + jsonPks.validate[Array[PublicKey]] match { + case s :JsSuccess[Array[PublicKey]] => + val pks = s.get + pksMap += (key -> pks) + case error : JsError => + throw new GetElectionInfoError(s"Error: public keys with invalid Array[PublicKey] format for election $key.\nPublic keys ${jsonPks.toString}\n") + } } // we need to generate vote_count encrypted ballots, fill the list with // random samples of the base list From 06f24048adce6d8fb7d6a9d934b258691384b353 Mon Sep 17 00:00:00 2001 From: Findeton Date: Fri, 17 Mar 2017 16:15:53 +0100 Subject: [PATCH 45/68] refactor pks map creation --- app/commands/Console.scala | 63 ++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index d13beeb3..434a0ab0 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -566,6 +566,42 @@ object Console { line } + /** + * Given a map of election ids and election info, it returns a map with the + * election public keys + */ + private def get_pks_map(electionsInfoMap: scala.collection.mutable.HashMap[Long, ElectionDTO]) + : scala.collection.mutable.HashMap[Long, Array[PublicKey]] = + { + val pksMap = scala.collection.mutable.HashMap[Long, Array[PublicKey]]() + // map election ids and public keys + electionsInfoMap foreach { case (key, value) => + val strPks = value.pks match { + case Some(strPks) => + strPks + case None => + val strDto = Json.toJson(value).toString + throw new GetElectionInfoError(s"Error: no public keys found for election $key. Election Info: $strDto") + } + val jsonPks = Try { + Json.parse(strPks) + } match { + case Success(jsonPks) => + jsonPks + case Failure(error) => + throw new GetElectionInfoError(s"Error: public keys with invalid JSON format for election $key.\nPublic keys: $strPks.\n${error.getMessage}") + } + jsonPks.validate[Array[PublicKey]] match { + case s :JsSuccess[Array[PublicKey]] => + val pks = s.get + pksMap += (key -> pks) + case error : JsError => + throw new GetElectionInfoError(s"Error: public keys with invalid Array[PublicKey] format for election $key.\nPublic keys ${jsonPks.toString}\n") + } + } + pksMap + } + /** * Given a list of plaintext ballots and a map of election ids and their * election info, it generates and encrypts the ballots, saving them to a @@ -580,36 +616,11 @@ object Console { { val promise = Promise[Unit]() Future { + val pksMap = get_pks_map(electionsInfoMap) // base list of encoded plaintext ballots val votes = ballotsList.par.map{ ballot : PlaintextBallot => (ballot.id, encodePlaintext( ballot, electionsInfoMap.get(ballot.id).get ) ) }.seq - // map election ids and public keys - val pksMap = scala.collection.mutable.HashMap[Long, Array[PublicKey]]() - electionsInfoMap foreach { case (key, value) => - val strPks = value.pks match { - case Some(strPks) => - strPks - case None => - val strDto = Json.toJson(value).toString - throw new GetElectionInfoError(s"Error: no public keys found for election $key. Election Info: $strDto") - } - val jsonPks = Try { - Json.parse(strPks) - } match { - case Success(jsonPks) => - jsonPks - case Failure(error) => - throw new GetElectionInfoError(s"Error: public keys with invalid JSON format for election $key.\nPublic keys: $strPks.\n${error.getMessage}") - } - jsonPks.validate[Array[PublicKey]] match { - case s :JsSuccess[Array[PublicKey]] => - val pks = s.get - pksMap += (key -> pks) - case error : JsError => - throw new GetElectionInfoError(s"Error: public keys with invalid Array[PublicKey] format for election $key.\nPublic keys ${jsonPks.toString}\n") - } - } // we need to generate vote_count encrypted ballots, fill the list with // random samples of the base list val toEncrypt : Seq[(Long, Array[Long])] = { From 6feee9a14f084ab5c747763b6c05f1a7181266dc Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 21 Mar 2017 12:27:10 +0100 Subject: [PATCH 46/68] change separator to | --- app/commands/Console.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 434a0ab0..b24e9825 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -305,14 +305,14 @@ object Console { * The format of each line of the ciphertexts file that this command reads * should be: * - * electionId-voterId-ballot + * electionId|voterId|ballot * - * Notice that the separator character is '-' + * Notice that the separator character is '|' * * The format of each line of the ciphertexts_khmac file that this command * creates is: * - * electionId-voterId-ballot-khmac + * electionId|voterId|ballot|khmac * * Notice that it simply adds the khmac. */ @@ -325,11 +325,11 @@ object Console { val writer = new FileWriter(Paths.get(ciphertexts_khmac_path)) promise completeWith { Future.traverse (io.Source.fromFile(ciphertexts_path).getLines()) { file_line => - val array = file_line.split('-') + val array = file_line.split('|') val eid = array(0).toLong val voterId = array(1) val khmac = get_khmac(voterId, "AuthEvent", eid, "vote", now) - writer.write(file_line + "-" + khmac) + writer.write(file_line + "|" + khmac) } map { _ => () } @@ -554,7 +554,7 @@ object Console { /** * Given an election id and an encrypted ballot, it generates a String text * line in the following format: - * electionId - voterId - vote + * electionId|voterId|vote * Notice that it automatically adds a random voter id */ private def generate_vote_line(id: Long, ballot: EncryptedVote) : String = { @@ -562,7 +562,7 @@ object Console { val voteHash = Crypto.sha256(vote) val voterId = generate_voterid() val voteDTO = Json.toJson(VoteDTO(vote, voteHash)).toString - val line = id.toString + "-" + generate_voterid() + "-" + voteDTO + "\n" + val line = id.toString + "|" + generate_voterid() + "|" + voteDTO + "\n" line } From b1c813ee0e50d10acd17fd6e2219c650c0537ba4 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 21 Mar 2017 12:46:21 +0100 Subject: [PATCH 47/68] fix bug: add newlines on add_khmac --- app/commands/Console.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index b24e9825..c36ff90f 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -329,7 +329,7 @@ object Console { val eid = array(0).toLong val voterId = array(1) val khmac = get_khmac(voterId, "AuthEvent", eid, "vote", now) - writer.write(file_line + "|" + khmac) + writer.write(file_line + "|" + khmac + "\n") } map { _ => () } @@ -695,7 +695,14 @@ object Console { System.exit(-1) } } else if ( "add_khmacs" == command) { - add_khmacs() + add_khmacs() onComplete { + case Success(value) => + println("add_khmacs success") + System.exit(0) + case Failure(error) => + println("add_khmacs error " + error) + System.exit(-1) + } } else { showHelp() } From dfd87e212c8e4202eaf64674ba434e9cfb5ab254 Mon Sep 17 00:00:00 2001 From: Findeton Date: Wed, 22 Mar 2017 13:28:24 +0100 Subject: [PATCH 48/68] add doc, add more options --- app/commands/Console.scala | 159 ++++++++++++++++++++++++++++++------- 1 file changed, 131 insertions(+), 28 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index c36ff90f..e39bc518 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -23,10 +23,10 @@ import utils.Crypto import java.util.concurrent.Executors import scala.concurrent._ -import javax.crypto.spec.SecretKeySpec import javax.crypto.Mac import javax.xml.bind.DatatypeConverter import java.math.BigInteger +import java.security.KeyStore import scala.concurrent.forkjoin._ import scala.collection.mutable.ArrayBuffer import scala.util.{Try, Success, Failure} @@ -94,8 +94,12 @@ object Console { // number of votes to create var vote_count : Long = 0 + // http or https + var http_type = "https" // hostname of the agora-elections server var host = "localhost" + // path to the service inside the host + var service_path = "elections/" // agora-elections port var port : Long = 9000 // default plaintext path @@ -108,6 +112,7 @@ object Console { // default voter id length (number of characters) var voterid_len : Int = 28 val voterid_alphabet: String = "0123456789abcdef" + var keystore_path: Option[String] = None // In order to make http requests with Play without a running Play instance, // we have to do this @@ -143,9 +148,24 @@ object Console { } else if ("--voterid-len" == args(arg_index + 1)) { voterid_len = args(arg_index + 2).toInt arg_index += 2 + } else if ("--ssl" == args(arg_index + 1)) { + if ("true" == args(arg_index + 2)) { + http_type = "https" + } else if ("false" == args(arg_index + 2)) { + http_type = "http" + } else { + throw new java.lang.IllegalArgumentException(s"Invalid --ssl option: " + args(arg_index + 2) + ". Valid values: true, false") + } + arg_index += 2 } else if ("--shared-secret" == args(arg_index + 1)) { shared_secret = args(arg_index + 2) arg_index += 2 + } else if ("--service-path" == args(arg_index + 1)) { + service_path = args(arg_index + 2) + arg_index += 2 + } else if ("--host" == args(arg_index + 1)) { + host = args(arg_index + 2) + arg_index += 2 } else if ("--port" == args(arg_index + 1)) { port = args(arg_index + 2).toLong if (port <= 0 || port > 65535) { @@ -163,29 +183,112 @@ object Console { */ private def showHelp() = { System.out.println( - """NAME - | commands.Console - Generate and send votes for benchmark purposes - | - |SYNOPSIS - | commands.Console [gen_votes|add_khmacs] [--vote-count number] - | [--plaintexts path] [--ciphertexts path] [--ciphertexts-khmac path] - | [--voterid-len number] [--shared-secret password] [--port port] - | - |DESCRIPTION - | In order to run this program, use: - | activator "runMain commands.Console [arguments]" - | possible commands: - | - gen_votes - | - add_khmacs - | Possible arguments - | - vote-count - | - plaintexts - | - ciphertexts - | - ciphertexts-khmac - | - voterid-len - | - shared-secret - | - port - |""".stripMargin) +"""NAME + | commands.Console - Generate and send votes for load testing and benchmarking + | + |SYNOPSIS + | commands.Console [gen_votes|add_khmacs] [--vote-count number] + | [--plaintexts path] [--ciphertexts path] [--ciphertexts-khmac path] + | [--voterid-len number] [--shared-secret password] [--port port] + | [--ssl boolean] [--host hostname] [--service-path path] + | + |DESCRIPTION + | In order to run this program, use: + | activator "runMain commands.Console [arguments]" + | + | Possible commands + | + | gen_votes + | Given a plaintext file where each string line contains a plaintext, + | it generates a ciphertext file with encrypted ballots. Encrypting + | ballots is a computationally intensive process, so it can take a + | while. The user ids are generated in a random fashion. + | + | add_khmacs + | Given a ciphertexts file where each string line contains an encrypted + | ballot, it adds a khmac to each line and saves the new file. This step + | is faster than encrypting the ballots and it prepares them to be sent + | to the ballot box with jmeter.The khmacs are only valid for a period + | of time and that's why they are generated just before sending the + | ballots. + | + | Possible arguments + | + | --vote-count + | Number of ballots to be generated. If the number of plaintexts is less + | than the vote count, then the plaintexts will be repeated till the + | desired number is achieved. + | Default value: 0 + | + | --plaintexts + | Path to the plaintexts file. In this file, each line represents a + | ballot with the chosen voting options. For example: + | 39||0|22,4 + | In this line, the election id is 39, on the first question the chosen + | option is a blank vote, on the first option it's voting to option 0, + | and on the last question it's voting to options 22 and 4. + | Notice that as each line indicates the election id, this file can be + | used to generate ballots for multiple elections. + | Default value: plaintexts.txt + | + | --ciphertexts + | The path to the ciphertexts file. This file is created by the + | gen_votes command and it contains the encrypted ballots generated from + | the plaintexts file. In this file, each line represents an encrypted + | ballot, in the following format: + | election_id|voter_id|ballot + | This is an intermediate file, because as it doesn't contain the + | khmacs, it's not ready to be used by jmeter. + | Default value: ciphertexts.csv + | + | --ciphertexts-khmac + | The path to the final ciphertexts file, which includes the khmacs. + | This file is generated by the add_khmacs command. The file format is + | the same as the ciphertexts file, except that it adds the khmac to + | each ballot: + | election_id|voter_id|ballot|khmac + | This is the file that will be fed to jmeter + | Default value: ciphertexts-khmac.csv + | + | --voterid-len + | Voter ids for each ballot are generated with random numbers and + | represented in hexadecimal format. This parameter configures the + | number of hexadecimal characters of the voter ids. + | Default value: 28 + | + | --shared-secret + | The shared secret required for authentication and creating khmacs. + | This value can be found on the config.yml + | agora.agora_elections.shared_secret variable. + | Default value: + | + | --service-path + | The location of service agora_elections in the host server. For + | example if a call to the server is: + | https://agora:443/elections/api/election/39 + | Then the service path is "elections/". + | Default value: elections/ + | + | --port + | Port number of the agora_elections service. + | Default value: 443 + | + | --host + | Hostname of the agora_elections service. + | Default value: localhost + | + | --ssl + | Enables or disables https. If true, http requests to the + | agora_elections service will use https. + | Usually, the agora_elections service will use a self-signed + | certificate. To make this program to accept the certificate, use + | -Djavax.net.ssl.trustStore=/path/to/jks/truststore and + | -Djavax.net.ssl.trustStorePassword=pass with a jks truststore that + | includes the agora_elections certificate. + | Possible values: true|false + | Default value: true + | + |""".stripMargin) } /** @@ -401,7 +504,7 @@ object Console { private def get_election_info(electionId: Long) : Future[ElectionDTO] = { val promise = Promise[ElectionDTO] Future { - val url = s"http://$host:$port/api/election/$electionId" + val url = s"$http_type://$host:$port/${service_path}api/election/$electionId" wsClient.url(url) .get() map { response => if(response.status == HTTP.OK) { val dto = (response.json \ "payload").validate[ElectionDTO].get @@ -452,7 +555,7 @@ object Console { val promise = Promise[Unit]() Future { val auth = get_khmac("", "AuthEvent", electionId, "edit") - val url = s"http://$host:$port/api/election/$electionId/dump-pks" + val url = s"$http_type://$host:$port/${service_path}api/election/$electionId/dump-pks" wsClient.url(url) .withHeaders("Authorization" -> auth) .post(Results.EmptyContent()).map { response => @@ -686,7 +789,7 @@ object Console { parse_args(args) val command = args(0) if ("gen_votes" == command) { - gen_votes() onComplete { + gen_votes() onComplete { case Success(value) => println("gen_votes success") System.exit(0) @@ -695,7 +798,7 @@ object Console { System.exit(-1) } } else if ( "add_khmacs" == command) { - add_khmacs() onComplete { + add_khmacs() onComplete { case Success(value) => println("add_khmacs success") System.exit(0) From 5ed005698fb4077493143ed3d8570840623b95d2 Mon Sep 17 00:00:00 2001 From: Findeton Date: Wed, 22 Mar 2017 16:30:10 +0100 Subject: [PATCH 49/68] change coding style --- app/commands/Console.scala | 575 +++++++++++++++++++++++++------------ 1 file changed, 388 insertions(+), 187 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index e39bc518..c28a390f 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -62,7 +62,8 @@ case class EncryptionError(message: String) extends Exception(message) * This object contains the states required for reading a plaintext ballot * It's used on Console.processPlaintextLine */ -object PlaintextBallot { +object PlaintextBallot +{ val ID = 0 // reading election ID val ANSWER = 1 // reading answers } @@ -70,12 +71,15 @@ object PlaintextBallot { /** * A simple class to write lines to a single file, in a multi-threading safe way */ -class FileWriter(path: Path) { +class FileWriter(path: Path) +{ Files.deleteIfExists(path) Files.createFile(path) - def write(content: String) : Future[Unit] = Future { - this.synchronized { + def write(content: String) : Future[Unit] = Future + { + this.synchronized + { Files.write(path, content.getBytes(StandardCharsets.UTF_8), APPEND) } } @@ -88,7 +92,8 @@ class FileWriter(path: Path) { * activator "run-main commands.Command " * from CLI */ -object Console { +object Console +{ //implicit val ec = ExecutionContext.fromExecutor(new ForkJoinPool(100)) //implicit val slickExecutionContext = Akka.system.dispatchers.lookup("play.akka.actor.slick-context") @@ -128,52 +133,81 @@ object Console { /** * Parse program arguments */ - private def parse_args(args: Array[String]) = { + private def parse_args(args: Array[String]) = + { var arg_index = 0 - while (arg_index + 2 < args.length) { + while (arg_index + 2 < args.length) + { // number of ballots to create - if ("--vote-count" == args(arg_index + 1)) { + if ("--vote-count" == args(arg_index + 1)) + { vote_count = args(arg_index + 2).toLong arg_index += 2 } - else if ("--plaintexts" == args(arg_index + 1)) { + else if ("--plaintexts" == args(arg_index + 1)) + { plaintexts_path = args(arg_index + 2) arg_index += 2 - } else if ("--ciphertexts" == args(arg_index + 1)) { + } + else if ("--ciphertexts" == args(arg_index + 1)) + { ciphertexts_path = args(arg_index + 2) arg_index += 2 - } else if ("--ciphertexts-khmac" == args(arg_index + 1)) { + } + else if ("--ciphertexts-khmac" == args(arg_index + 1)) + { ciphertexts_khmac_path = args(arg_index + 2) arg_index += 2 - } else if ("--voterid-len" == args(arg_index + 1)) { + } + else if ("--voterid-len" == args(arg_index + 1)) + { voterid_len = args(arg_index + 2).toInt arg_index += 2 - } else if ("--ssl" == args(arg_index + 1)) { - if ("true" == args(arg_index + 2)) { + } + else if ("--ssl" == args(arg_index + 1)) + { + if ("true" == args(arg_index + 2)) + { http_type = "https" - } else if ("false" == args(arg_index + 2)) { + } + else if ("false" == args(arg_index + 2)) + { http_type = "http" - } else { + } + else + { throw new java.lang.IllegalArgumentException(s"Invalid --ssl option: " + args(arg_index + 2) + ". Valid values: true, false") } arg_index += 2 - } else if ("--shared-secret" == args(arg_index + 1)) { + } + else if ("--shared-secret" == args(arg_index + 1)) + { shared_secret = args(arg_index + 2) arg_index += 2 - } else if ("--service-path" == args(arg_index + 1)) { + } + else if ("--service-path" == args(arg_index + 1)) + { service_path = args(arg_index + 2) arg_index += 2 - } else if ("--host" == args(arg_index + 1)) { + } + else if ("--host" == args(arg_index + 1)) + { host = args(arg_index + 2) arg_index += 2 - } else if ("--port" == args(arg_index + 1)) { + } + else if ("--port" == args(arg_index + 1)) + { port = args(arg_index + 2).toLong - if (port <= 0 || port > 65535) { + if (port <= 0 || port > 65535) + { throw new java.lang.IllegalArgumentException(s"Invalid port $port") } arg_index += 2 - } else { - throw new java.lang.IllegalArgumentException("Unrecognized argument: " + args(arg_index + 1)) + } + else + { + throw new java.lang.IllegalArgumentException("Unrecognized argument: " + + args(arg_index + 1)) } } } @@ -181,7 +215,8 @@ object Console { /** * Prints to the standard output how to use this program */ - private def showHelp() = { + private def showHelp() = + { System.out.println( """NAME | commands.Console - Generate and send votes for load testing and benchmarking @@ -304,7 +339,11 @@ object Console { * - A blank vote for an answer is represented with no characters * - For example '||' means a blank vote for the corresponding answer */ - private def processPlaintextLine(line: String, lineNumber: Long) : PlaintextBallot = { + private def processPlaintextLine( + line: String, + lineNumber: Long) + : PlaintextBallot = + { // in this variable we keep adding the characters that will form a complete number var strIndex: Option[String] = None // state: either reading the election index or the election answers @@ -318,10 +357,14 @@ object Console { var answersBuffer: ArrayBuffer[PlaintextAnswer] = ArrayBuffer[PlaintextAnswer]() // iterate through all characters in the string line - for (i <- 0 until line.length) { + for (i <- 0 until line.length) + { val c = line.charAt(i) - if(c.isDigit) { // keep reading digits till we get the whole number - strIndex match { + // keep reading digits till we get the whole number + if(c.isDigit) + { + strIndex match + { // add a character to the string containing a number case Some(strIndexValue) => strIndex = Some(strIndexValue + c.toString) @@ -331,13 +374,17 @@ object Console { } } // it's not a digit - else { + else + { // state: reading election ID - if (PlaintextBallot.ID == state) { - if ('|' != c) { + if (PlaintextBallot.ID == state) + { + if ('|' != c) + { throw PlaintextError(s"Error on line $lineNumber, character $i: character separator '|' not found after election index . Line: $line") } - strIndex match { + strIndex match + { case Some(strIndexValue) => ballot = new PlaintextBallot(strIndex.get.toLong, ballot.answers) strIndex = None @@ -348,12 +395,16 @@ object Console { } } // state: reading answers to each question - else if (PlaintextBallot.ANSWER == state) { - optionsBuffer match { + else if (PlaintextBallot.ANSWER == state) + { + optionsBuffer match + { case Some(optionsBufferValue) => // end of this question, add question to buffer - if ('|' == c) { - if (strIndex.isDefined) { + if ('|' == c) + { + if (strIndex.isDefined) + { optionsBufferValue += strIndex.get.toLong strIndex = None } @@ -361,15 +412,19 @@ object Console { optionsBuffer = Some(ArrayBuffer[Long]()) } // add chosen option to buffer - else if(',' == c) { - strIndex match { + else if(',' == c) + { + strIndex match + { case Some(strIndexValue) => optionsBufferValue += strIndexValue.toLong strIndex = None case None => throw PlaintextError(s"Error on line $lineNumber, character $i: option number not recognized before comma on question ${ballot.answers.length}. Line: $line") } - } else { + } + else + { throw PlaintextError(s"Error on line $lineNumber, character $i: invalid character: $c. Line: $line") } case None => @@ -379,10 +434,12 @@ object Console { } } // add the last answer - optionsBuffer match { + optionsBuffer match + { case Some(optionsBufferValue) => // read last option of last answer, if there's any - if (strIndex.isDefined) { + if (strIndex.isDefined) + { optionsBufferValue += strIndex.get.toLong } answersBuffer += PlaintextAnswer(optionsBufferValue.toArray) @@ -419,29 +476,41 @@ object Console { * * Notice that it simply adds the khmac. */ - private def add_khmacs() : Future[Unit] = { + private def add_khmacs() : Future[Unit] = + { val promise = Promise[Unit]() - Future { + Future + { // check that the file we want to read exists - if (Files.exists(Paths.get(ciphertexts_path))) { + if (Files.exists(Paths.get(ciphertexts_path))) + { val now = Some(System.currentTimeMillis / 1000) val writer = new FileWriter(Paths.get(ciphertexts_khmac_path)) - promise completeWith { - Future.traverse (io.Source.fromFile(ciphertexts_path).getLines()) { file_line => - val array = file_line.split('|') - val eid = array(0).toLong - val voterId = array(1) - val khmac = get_khmac(voterId, "AuthEvent", eid, "vote", now) - writer.write(file_line + "|" + khmac + "\n") - } map { _ => - () + promise completeWith + { + Future.traverse (io.Source.fromFile(ciphertexts_path).getLines()) + { + file_line => + val array = file_line.split('|') + val eid = array(0).toLong + val voterId = array(1) + val khmac = get_khmac(voterId, "AuthEvent", eid, "vote", now) + writer.write(file_line + "|" + khmac + "\n") + } + map + { + _ => () } } - } else { + } + else + { throw new java.io.FileNotFoundException("tally does not exist") } - } recover { case error: Throwable => - promise failure error + } + recover + { + case error: Throwable => promise failure error } promise.future } @@ -451,34 +520,48 @@ object Console { * See Console.processPlaintextLine() comments for more info on the format * of the plaintext file. */ - private def parsePlaintexts(): Future[(scala.collection.immutable.List[PlaintextBallot], scala.collection.immutable.Set[Long])] = { + private def parsePlaintexts() + : Future[ + (scala.collection.immutable.List[PlaintextBallot], + scala.collection.immutable.Set[Long])] = + { val promise = Promise[(scala.collection.immutable.List[PlaintextBallot], scala.collection.immutable.Set[Long])]() - Future { + Future + { val path = Paths.get(plaintexts_path) // Check that the plaintexts file exists - if (Files.exists(path)) { + if (Files.exists(path)) + { // buffer with all the parsed ballots val ballotsList = scala.collection.mutable.ListBuffer[PlaintextBallot]() // set of all the election ids val electionsSet = scala.collection.mutable.LinkedHashSet[Long]() // read all lines - io.Source.fromFile(plaintexts_path).getLines().zipWithIndex.foreach { + io.Source.fromFile(plaintexts_path).getLines().zipWithIndex.foreach + { case (line, number) => // parse line val ballot = processPlaintextLine(line, number) ballotsList += ballot electionsSet += ballot.id } - if ( electionsSet.isEmpty || ballotsList.isEmpty ) { + if ( electionsSet.isEmpty || ballotsList.isEmpty ) + { throw PlaintextError("Error: no ballot found") - } else { + } + else + { promise success ( ( ballotsList.sortBy(_.id).toList, electionsSet.toSet ) ) } - } else { + } + else + { throw new java.io.FileNotFoundException(s"plaintext file ${path.toAbsolutePath.toString} does not exist or can't be opened") } - } recover { case error: Throwable => - promise failure error + } + recover + { + case error: Throwable => promise failure error } promise.future } @@ -486,8 +569,17 @@ object Console { /** * Generate khmac */ - private def get_khmac(userId: String, objType: String, objId: Long, perm: String, nowOpt: Option[Long] = None) : String = { - val now: Long = nowOpt match { + private def get_khmac( + userId: String, + objType: String, + objId: Long, + perm: String, + nowOpt: Option[Long] = None + ) + : String = + { + val now: Long = nowOpt match + { case Some(time) => time case None => System.currentTimeMillis / 1000 } @@ -501,22 +593,34 @@ object Console { * Makes an http request to agora-elections to get the election info. * Returns the parsed election info */ - private def get_election_info(electionId: Long) : Future[ElectionDTO] = { + private def get_election_info(electionId: Long) : Future[ElectionDTO] = + { val promise = Promise[ElectionDTO] - Future { + Future + { val url = s"$http_type://$host:$port/${service_path}api/election/$electionId" - wsClient.url(url) .get() map { response => - if(response.status == HTTP.OK) { - val dto = (response.json \ "payload").validate[ElectionDTO].get - promise success dto - } else { - promise failure GetElectionInfoError(s"HTTP GET request to $url returned status: ${response.status} and body: ${response.body}") - } - } recover { case error: Throwable => - promise failure error + wsClient.url(url) .get() map + { + response => + if(response.status == HTTP.OK) + { + val dto = (response.json \ "payload").validate[ElectionDTO].get + promise success dto + } + else + { + promise failure GetElectionInfoError( + s"HTTP GET request to $url returned status: ${response.status} and body: ${response.body}") + } } - } recover { case error: Throwable => - promise failure error + recover + { + case error: Throwable => promise failure error + } + } + recover + { + case error: Throwable => promise failure error } promise.future } @@ -525,24 +629,39 @@ object Console { * Given a set of election ids, it returns a map of the election ids and their * election info. */ - private def get_election_info_all(electionsSet: scala.collection.immutable.Set[Long]) : Future[scala.collection.mutable.HashMap[Long, ElectionDTO]] = { + private def get_election_info_all( + electionsSet: scala.collection.immutable.Set[Long] + ) + : Future[scala.collection.mutable.HashMap[Long, ElectionDTO]] = + { val promise = Promise[scala.collection.mutable.HashMap[Long, ElectionDTO]]() - Future { + Future + { val map = scala.collection.mutable.HashMap[Long, ElectionDTO]() - promise completeWith { - Future.traverse(electionsSet) { eid => - get_election_info(eid) map { dto : ElectionDTO => - // using synchronized to make it thread-safe - this.synchronized { - map += (dto.id -> dto) + promise completeWith + { + Future.traverse(electionsSet) + { + eid => + get_election_info(eid) map + { + dto : ElectionDTO => + // using synchronized to make it thread-safe + this.synchronized + { + map += (dto.id -> dto) + } } - } - } map { _ => - map + } + map + { + _ => map } } - } recover { case error: Throwable => - promise failure error + } + recover + { + case error: Throwable => promise failure error } promise.future } @@ -551,24 +670,35 @@ object Console { * Makes an HTTP request to agora-elections to dump the public keys for a * given election id. */ - private def dump_pks(electionId: Long): Future[Unit] = { + private def dump_pks(electionId: Long): Future[Unit] = + { val promise = Promise[Unit]() - Future { + Future + { val auth = get_khmac("", "AuthEvent", electionId, "edit") val url = s"$http_type://$host:$port/${service_path}api/election/$electionId/dump-pks" wsClient.url(url) .withHeaders("Authorization" -> auth) - .post(Results.EmptyContent()).map { response => - if(response.status == HTTP.OK) { - promise success ( () ) - } else { - promise failure DumpPksError(s"HTTP POST request to $url returned status: ${response.status} and body: ${response.body}") - } - } recover { case error: Throwable => - promise failure error + .post(Results.EmptyContent()).map + { + response => + if(response.status == HTTP.OK) + { + promise success ( () ) + } + else + { + promise failure DumpPksError(s"HTTP POST request to $url returned status: ${response.status} and body: ${response.body}") + } + } + recover + { + case error: Throwable => promise failure error } - } recover { case error: Throwable => - promise failure error + } + recover + { + case error: Throwable => promise failure error } promise.future } @@ -577,16 +707,25 @@ object Console { * Given a set of election ids, it returns a future that will be completed * when all the public keys of all those elections have been dumped. */ - private def dump_pks_elections(electionsSet: scala.collection.immutable.Set[Long]): Future[Unit] = { + private def dump_pks_elections( + electionsSet: scala.collection.immutable.Set[Long] + ) + : Future[Unit] = + { val promise = Promise[Unit]() - Future { - promise completeWith { - Future.traverse( electionsSet ) ( x => dump_pks(x) ) map { _ => - () + Future + { + promise completeWith + { + Future.traverse( electionsSet ) ( x => dump_pks(x) ) map + { + _ => () } } - } recover { case error: Throwable => - promise failure error + } + recover + { + case error: Throwable => promise failure error } promise.future } @@ -595,9 +734,15 @@ object Console { * Given a plaintext and the election info, it encodes each question's answers * into a number, ready to be encrypted */ - private def encodePlaintext(ballot: PlaintextBallot, dto: ElectionDTO): Array[Long] = { + private def encodePlaintext( + ballot: PlaintextBallot, + dto: ElectionDTO + ) + : Array[Long] = + { var array = new Array[Long](ballot.answers.length) - if (dto.configuration.questions.length != ballot.answers.length) { + if (dto.configuration.questions.length != ballot.answers.length) + { val strBallot = Json.toJson(ballot).toString val strDto = Json.toJson(dto).toString throw EncodePlaintextError( @@ -605,13 +750,16 @@ object Console { |Error: wrong number of questions on the plaintext ballot: |${dto.configuration.questions.length} != ${ballot.answers.length}""") } - for (i <- 0 until array.length) { - Try { + for (i <- 0 until array.length) + { + Try + { val numChars = ( dto.configuration.questions(i).answers.length + 2 ).toString.length // holds the value of the encoded answer, before converting it to a Long var strValue : String = "" val answer = ballot.answers(i) - for (j <- 0 until answer.options.length) { + for (j <- 0 until answer.options.length) + { // sum 1 as the encryption method can't encode value zero val optionStrBase = ( answer.options(j) + 1 ).toString // Each chosen option needs to have the same length in number of @@ -619,14 +767,17 @@ object Console { strValue += "0" * (numChars - optionStrBase.length) + optionStrBase } // blank vote - if (0 == answer.options.length) { + if (0 == answer.options.length) + { val optionStrBase = ( dto.configuration.questions(i).answers.length + 2 ).toString strValue += "0" * (numChars - optionStrBase.length) + optionStrBase } // Convert to long. Notice that the zeros on the left added by the last // chosen option won't be included array(i) = strValue.toLong - } match { + } + match + { case Failure(error) => val strBallot = Json.toJson(ballot).toString val strDto = Json.toJson(dto).toString @@ -641,16 +792,19 @@ object Console { * Generate a random string with a given length and alphabet * Note: This is not truly random, it uses a pseudo-random generator */ - private def gen_rnd_str(len: Int, choices: String) : String = { - (0 until len) map { _ => - choices( scala.util.Random.nextInt(choices.length) ) + private def gen_rnd_str(len: Int, choices: String) : String = + { + (0 until len) map + { + _ => choices( scala.util.Random.nextInt(choices.length) ) } mkString } /** * Generate a random voter id */ - private def generate_voterid() : String = { + private def generate_voterid() : String = + { gen_rnd_str(voterid_len, voterid_alphabet) } @@ -660,7 +814,8 @@ object Console { * electionId|voterId|vote * Notice that it automatically adds a random voter id */ - private def generate_vote_line(id: Long, ballot: EncryptedVote) : String = { + private def generate_vote_line(id: Long, ballot: EncryptedVote) : String = + { val vote = Json.toJson(ballot).toString val voteHash = Crypto.sha256(vote) val voterId = generate_voterid() @@ -673,34 +828,43 @@ object Console { * Given a map of election ids and election info, it returns a map with the * election public keys */ - private def get_pks_map(electionsInfoMap: scala.collection.mutable.HashMap[Long, ElectionDTO]) - : scala.collection.mutable.HashMap[Long, Array[PublicKey]] = + private def get_pks_map( + electionsInfoMap: scala.collection.mutable.HashMap[Long, ElectionDTO] + ) + : scala.collection.mutable.HashMap[Long, Array[PublicKey]] = { val pksMap = scala.collection.mutable.HashMap[Long, Array[PublicKey]]() // map election ids and public keys - electionsInfoMap foreach { case (key, value) => - val strPks = value.pks match { - case Some(strPks) => - strPks - case None => - val strDto = Json.toJson(value).toString - throw new GetElectionInfoError(s"Error: no public keys found for election $key. Election Info: $strDto") - } - val jsonPks = Try { - Json.parse(strPks) - } match { - case Success(jsonPks) => - jsonPks - case Failure(error) => - throw new GetElectionInfoError(s"Error: public keys with invalid JSON format for election $key.\nPublic keys: $strPks.\n${error.getMessage}") - } - jsonPks.validate[Array[PublicKey]] match { - case s :JsSuccess[Array[PublicKey]] => - val pks = s.get - pksMap += (key -> pks) - case error : JsError => - throw new GetElectionInfoError(s"Error: public keys with invalid Array[PublicKey] format for election $key.\nPublic keys ${jsonPks.toString}\n") - } + electionsInfoMap foreach + { + case (key, value) => + val strPks = value.pks match + { + case Some(strPks) => + strPks + case None => + val strDto = Json.toJson(value).toString + throw new GetElectionInfoError(s"Error: no public keys found for election $key. Election Info: $strDto") + } + val jsonPks = Try + { + Json.parse(strPks) + } + match + { + case Success(jsonPks) => + jsonPks + case Failure(error) => + throw new GetElectionInfoError(s"Error: public keys with invalid JSON format for election $key.\nPublic keys: $strPks.\n${error.getMessage}") + } + jsonPks.validate[Array[PublicKey]] match + { + case s :JsSuccess[Array[PublicKey]] => + val pks = s.get + pksMap += (key -> pks) + case error : JsError => + throw new GetElectionInfoError(s"Error: public keys with invalid Array[PublicKey] format for election $key.\nPublic keys ${jsonPks.toString}\n") + } } pksMap } @@ -718,41 +882,56 @@ object Console { : Future[Unit] = { val promise = Promise[Unit]() - Future { + Future + { val pksMap = get_pks_map(electionsInfoMap) // base list of encoded plaintext ballots - val votes = ballotsList.par.map{ ballot : PlaintextBallot => - (ballot.id, encodePlaintext( ballot, electionsInfoMap.get(ballot.id).get ) ) + val votes = ballotsList.par.map + { + ballot : PlaintextBallot => + (ballot.id, encodePlaintext( ballot, electionsInfoMap.get(ballot.id).get ) ) }.seq // we need to generate vote_count encrypted ballots, fill the list with // random samples of the base list - val toEncrypt : Seq[(Long, Array[Long])] = { + val toEncrypt : Seq[(Long, Array[Long])] = + { val extraSize = vote_count - votes.length val extra = Array.fill(extraSize.toInt){ votes(scala.util.Random.nextInt(votes.length)) } votes ++ extra } val writer = new FileWriter(Paths.get(ciphertexts_path)) - promise completeWith { - Future.traverse (toEncrypt) { case (electionId, plaintext) => - val writePromise = Promise[Unit]() - Future { - val pks = pksMap.get(electionId).get - if (pks.length != plaintext.length) { - throw new EncryptionError(s"${pks.length} != ${plaintext.length}") + promise completeWith + { + Future.traverse (toEncrypt) + { + case (electionId, plaintext) => + val writePromise = Promise[Unit]() + Future + { + val pks = pksMap.get(electionId).get + if (pks.length != plaintext.length) + { + throw new EncryptionError(s"${pks.length} != ${plaintext.length}") + } + val encryptedVote = Crypto.encrypt(pks, plaintext) + val line = generate_vote_line(electionId, encryptedVote) + writePromise completeWith writer.write(line) } - val encryptedVote = Crypto.encrypt(pks, plaintext) - val line = generate_vote_line(electionId, encryptedVote) - writePromise completeWith writer.write(line) - } recover { case error: Throwable => - writePromise failure error - } - writePromise.future - } map { _ => - () + recover + { + case error: Throwable => writePromise failure error + } + writePromise.future + } + map + { + _ => () } } - } recover { case error: Throwable => - promise failure error + } + recover + { + case error: Throwable => promise failure error } promise.future } @@ -760,36 +939,49 @@ object Console { /** * Given a list of plaintexts, it generates their ciphertexts */ - private def gen_votes(): Future[Unit] = { + private def gen_votes(): Future[Unit] = + { val promise = Promise[Unit]() - Future { - promise completeWith { - parsePlaintexts() flatMap { + Future + { + promise completeWith + { + parsePlaintexts() flatMap + { case (ballotsList, electionsSet) => val dumpPksFuture = dump_pks_elections(electionsSet) - get_election_info_all(electionsSet) flatMap { + get_election_info_all(electionsSet) flatMap + { electionsInfoMap => - dumpPksFuture flatMap { + dumpPksFuture flatMap + { pksDumped => encryptBallots(ballotsList, electionsInfoMap) } } } } - } recover { case error: Throwable => - promise failure error + } + recover + { + case error: Throwable => promise failure error } promise.future } - def main(args: Array[String]) : Unit = { - if(0 == args.length) { + def main(args: Array[String]) : Unit = + { + if(0 == args.length) + { showHelp() - } else { + } else + { parse_args(args) val command = args(0) - if ("gen_votes" == command) { - gen_votes() onComplete { + if ("gen_votes" == command) + { + gen_votes() onComplete + { case Success(value) => println("gen_votes success") System.exit(0) @@ -797,8 +989,11 @@ object Console { println("gen_votes error " + error) System.exit(-1) } - } else if ( "add_khmacs" == command) { - add_khmacs() onComplete { + } + else if ("add_khmacs" == command) + { + add_khmacs() onComplete + { case Success(value) => println("add_khmacs success") System.exit(0) @@ -806,7 +1001,13 @@ object Console { println("add_khmacs error " + error) System.exit(-1) } - } else { + } + else if ("gen_plaintexts" == command) + { + + } + else + { showHelp() } } From 652b38e009d17eb1ef46ef5e8345d24db253d4f8 Mon Sep 17 00:00:00 2001 From: Findeton Date: Mon, 27 Mar 2017 20:58:47 +0200 Subject: [PATCH 50/68] adding gen_plaintexts code --- app/commands/Console.scala | 223 +++++++++++++++++++++++++++++++++---- 1 file changed, 203 insertions(+), 20 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index c28a390f..adeadf26 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -57,6 +57,7 @@ case class BallotEncryptionError(message: String) extends Exception(message) case class GetElectionInfoError(message: String) extends Exception(message) case class EncodePlaintextError(message: String) extends Exception(message) case class EncryptionError(message: String) extends Exception(message) +case class ElectionIdsFileError(message: String) extends Exception(message) /** * This object contains the states required for reading a plaintext ballot @@ -118,6 +119,7 @@ object Console var voterid_len : Int = 28 val voterid_alphabet: String = "0123456789abcdef" var keystore_path: Option[String] = None + var election_ids_path : Option[String] = None // In order to make http requests with Play without a running Play instance, // we have to do this @@ -190,6 +192,11 @@ object Console service_path = args(arg_index + 2) arg_index += 2 } + else if ("--election-ids" == args(arg_index + 1)) + { + election_ids_path = Some(args(arg_index + 2)) + arg_index += 2 + } else if ("--host" == args(arg_index + 1)) { host = args(arg_index + 2) @@ -226,6 +233,7 @@ object Console | [--plaintexts path] [--ciphertexts path] [--ciphertexts-khmac path] | [--voterid-len number] [--shared-secret password] [--port port] | [--ssl boolean] [--host hostname] [--service-path path] + | [--election-ids path] | |DESCRIPTION | In order to run this program, use: @@ -233,6 +241,11 @@ object Console | | Possible commands | + | gen_plaintexts + | Generates plaintext ballots covering every option for every question + | for each election id. Use --vote-count to set the number of plaintexts + | to be generated, which should be at least the number of elections. + | | gen_votes | Given a plaintext file where each string line contains a plaintext, | it generates a ciphertext file with encrypted ballots. Encrypting @@ -323,6 +336,10 @@ object Console | Possible values: true|false | Default value: true | + | --election-ids path + | Path to a file containing the election ids. Required parameter by + | command gen_plaintexts. + | |""".stripMargin) } @@ -497,7 +514,7 @@ object Console val khmac = get_khmac(voterId, "AuthEvent", eid, "vote", now) writer.write(file_line + "|" + khmac + "\n") } - map + .map { _ => () } @@ -508,7 +525,7 @@ object Console throw new java.io.FileNotFoundException("tally does not exist") } } - recover + .recover { case error: Throwable => promise failure error } @@ -537,7 +554,7 @@ object Console // set of all the election ids val electionsSet = scala.collection.mutable.LinkedHashSet[Long]() // read all lines - io.Source.fromFile(plaintexts_path).getLines().zipWithIndex.foreach + io.Source.fromFile(plaintexts_path).getLines().zipWithIndex.foreach { case (line, number) => // parse line @@ -559,7 +576,7 @@ object Console throw new java.io.FileNotFoundException(s"plaintext file ${path.toAbsolutePath.toString} does not exist or can't be opened") } } - recover + .recover { case error: Throwable => promise failure error } @@ -599,7 +616,7 @@ object Console Future { val url = s"$http_type://$host:$port/${service_path}api/election/$electionId" - wsClient.url(url) .get() map + wsClient.url(url) .get() .map { response => if(response.status == HTTP.OK) @@ -613,12 +630,12 @@ object Console s"HTTP GET request to $url returned status: ${response.status} and body: ${response.body}") } } - recover + .recover { case error: Throwable => promise failure error } } - recover + .recover { case error: Throwable => promise failure error } @@ -653,13 +670,13 @@ object Console } } } - map + .map { _ => map } } } - recover + .recover { case error: Throwable => promise failure error } @@ -691,12 +708,12 @@ object Console promise failure DumpPksError(s"HTTP POST request to $url returned status: ${response.status} and body: ${response.body}") } } - recover + .recover { case error: Throwable => promise failure error } } - recover + .recover { case error: Throwable => promise failure error } @@ -723,7 +740,7 @@ object Console } } } - recover + .recover { case error: Throwable => promise failure error } @@ -889,7 +906,7 @@ object Console val votes = ballotsList.par.map { ballot : PlaintextBallot => - (ballot.id, encodePlaintext( ballot, electionsInfoMap.get(ballot.id).get ) ) + (ballot.id, encodePlaintext( ballot, electionsInfoMap.get(ballot.id).get ) ) }.seq // we need to generate vote_count encrypted ballots, fill the list with // random samples of the base list @@ -917,19 +934,19 @@ object Console val line = generate_vote_line(electionId, encryptedVote) writePromise completeWith writer.write(line) } - recover + .recover { case error: Throwable => writePromise failure error } writePromise.future } - map + .map { _ => () } } } - recover + .recover { case error: Throwable => promise failure error } @@ -939,7 +956,7 @@ object Console /** * Given a list of plaintexts, it generates their ciphertexts */ - private def gen_votes(): Future[Unit] = + private def gen_votes() : Future[Unit] = { val promise = Promise[Unit]() Future @@ -962,7 +979,165 @@ object Console } } } - recover + .recover + { + case error: Throwable => promise failure error + } + promise.future + } + + private def generate_plaintexts_for_eid(eid: Long, numVotes: Long, dto: ElectionDTO) : String = + { + var outText: String = "" + val sEid = eid.toString + for (i <- 0 until numVotes.toInt) + { + var line: String = sEid + for (question <- dto.configuration.questions) + { + line += "|" + val diff = question.max - question.min + val optionsBuffer : scala.collection.mutable.ArrayBuffer[Long] = + (0 until question.answers.size) + .map(_.toLong).to[scala.collection.mutable.ArrayBuffer] + val num_answers = question.min + scala.util.Random.nextInt(diff + 1) + for (j <- 0 until num_answers) + { + if (0 != j) + { + line += "," + } + val option = optionsBuffer(scala.util.Random.nextInt(optionsBuffer.size)) + optionsBuffer -= option + line += option.toString + } + } + outText += line + } + outText + } + + private def generate_map_num_votes_dto( + electionsInfoMap: scala.collection.mutable.HashMap[Long, ElectionDTO] + ) + : scala.collection.mutable.HashMap[Long, (Long, ElectionDTO)] + = + { + val numElections = electionsInfoMap.size + if (0 == vote_count) + { + throw new java.lang.IllegalArgumentException(s"Missing argument: --vote-count") + } + else if (vote_count < numElections) + { + throw new java.lang.IllegalArgumentException( + s"vote-count: $vote_count is less than the number of elections ($numElections)") + } + // minimum number of votes per election + val votesFloor = (vote_count.toFloat / numElections.toFloat).floor.toLong + var votesSoFar: Long = 0 + var electionsSoFar: Long = 0 + val electionsVotesDtoMap = scala.collection.mutable.HashMap[Long, (Long, ElectionDTO)]() + electionsInfoMap.foreach + { + case (eid, dto) => + val votesToGenThisEid = if (vote_count - votesSoFar > votesFloor*(numElections - electionsSoFar)) + { + votesFloor + 1 + } + else + { + votesFloor + } + electionsVotesDtoMap += (eid -> (votesToGenThisEid, dto)) + votesSoFar += votesToGenThisEid + electionsSoFar += 1 + } + electionsVotesDtoMap + } + + private def generate_save_plaintexts( + electionsInfoMap: scala.collection.mutable.HashMap[Long, ElectionDTO] + ) + : Future[Unit] = + { + val promise = Promise[Unit]() + Future + { + val electionsVotesDtoMap = generate_map_num_votes_dto(electionsInfoMap) + val writer = new FileWriter(Paths.get(plaintexts_path)) + promise completeWith { + Future.traverse (electionsVotesDtoMap) + { + case (eid, (numVotes, dto)) => + val writePromise = Promise[Unit]() + Future + { + val text = generate_plaintexts_for_eid(eid, numVotes, dto) + writePromise completeWith writer.write(text) + } + .recover + { + case error: Throwable => writePromise failure error + } + writePromise.future + }.map + { + _ => () + } + } + } + .recover + { + case error: Throwable => promise failure error + } + promise.future + } + + private def getElectionsSet() : Future[scala.collection.immutable.Set[Long]] = + { + val promise = Promise[scala.collection.immutable.Set[Long]]() + Future + { + if (election_ids_path.isEmpty) { + throw new java.lang.IllegalArgumentException(s"Missing argument: --elections-ids") + } + val eids_path = Paths.get(election_ids_path.get) + val electionsSet = + io.Source.fromFile(plaintexts_path).getLines().toList.map + { + strEid => strEid.toLong + }.toSet + promise.success(electionsSet) + } + .recover + { + case error: Throwable => promise failure error + } + promise.future + } + + /** + * + */ + private def gen_plaintexts() : Future[Unit] = + { + val promise = Promise[Unit]() + Future + { + promise completeWith + { + getElectionsSet() flatMap + { + electionsSet => + get_election_info_all(electionsSet) flatMap { + electionsInfoMap => + generate_save_plaintexts(electionsInfoMap) + } + } + } + } + .recover { case error: Throwable => promise failure error } @@ -971,7 +1146,7 @@ object Console def main(args: Array[String]) : Unit = { - if(0 == args.length) + if(0 == args.length) { showHelp() } else @@ -1004,7 +1179,15 @@ object Console } else if ("gen_plaintexts" == command) { - + gen_plaintexts() onComplete + { + case Success(value) => + println("gen_plaintexts success") + System.exit(0) + case Failure(error) => + println("gen_plaintexts error " + error) + System.exit(-1) + } } else { From ca1b7db3c7cf8c6e08ed20d3e4e12a81ede63dde Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 28 Mar 2017 10:43:56 +0200 Subject: [PATCH 51/68] fix error: getElectionsSet wasn't opening election ids file, also syntax improvements --- app/commands/Console.scala | 48 +++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index adeadf26..931ab579 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -119,7 +119,7 @@ object Console var voterid_len : Int = 28 val voterid_alphabet: String = "0123456789abcdef" var keystore_path: Option[String] = None - var election_ids_path : Option[String] = None + var election_ids_path : String = "election_ids.txt" // In order to make http requests with Play without a running Play instance, // we have to do this @@ -194,7 +194,7 @@ object Console } else if ("--election-ids" == args(arg_index + 1)) { - election_ids_path = Some(args(arg_index + 2)) + election_ids_path = args(arg_index + 2) arg_index += 2 } else if ("--host" == args(arg_index + 1)) @@ -990,7 +990,7 @@ object Console { var outText: String = "" val sEid = eid.toString - for (i <- 0 until numVotes.toInt) + for (i <- 0L until numVotes) { var line: String = sEid for (question <- dto.configuration.questions) @@ -998,8 +998,8 @@ object Console line += "|" val diff = question.max - question.min val optionsBuffer : scala.collection.mutable.ArrayBuffer[Long] = - (0 until question.answers.size) - .map(_.toLong).to[scala.collection.mutable.ArrayBuffer] + (0L until question.answers.size.toLong) + .to[scala.collection.mutable.ArrayBuffer] val num_answers = question.min + scala.util.Random.nextInt(diff + 1) for (j <- 0 until num_answers) { @@ -1094,20 +1094,21 @@ object Console promise.future } - private def getElectionsSet() : Future[scala.collection.immutable.Set[Long]] = + private def getElectionsSet() + : Future[scala.collection.immutable.Set[Long]] = { val promise = Promise[scala.collection.immutable.Set[Long]]() Future { - if (election_ids_path.isEmpty) { - throw new java.lang.IllegalArgumentException(s"Missing argument: --elections-ids") - } - val eids_path = Paths.get(election_ids_path.get) + val eids_path = Paths.get(election_ids_path) val electionsSet = - io.Source.fromFile(plaintexts_path).getLines().toList.map - { - strEid => strEid.toLong - }.toSet + io.Source.fromFile(election_ids_path) + .getLines() + .toList.map + { + strEid => strEid.toLong + } + .toSet promise.success(electionsSet) } .recover @@ -1120,20 +1121,22 @@ object Console /** * */ - private def gen_plaintexts() : Future[Unit] = + private def gen_plaintexts() + : Future[Unit] = { val promise = Promise[Unit]() Future { promise completeWith { - getElectionsSet() flatMap + getElectionsSet() + .flatMap { - electionsSet => - get_election_info_all(electionsSet) flatMap { - electionsInfoMap => - generate_save_plaintexts(electionsInfoMap) - } + electionsSet => get_election_info_all(electionsSet) + } + .flatMap + { + electionsInfoMap => generate_save_plaintexts(electionsInfoMap) } } } @@ -1144,7 +1147,8 @@ object Console promise.future } - def main(args: Array[String]) : Unit = + def main(args: Array[String]) + : Unit = { if(0 == args.length) { From 5db08c242b86664ef06181aeac49517576470f7e Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 28 Mar 2017 14:13:38 +0200 Subject: [PATCH 52/68] fix ballot encryption for questions with many possible selected options, add comments --- app/commands/Console.scala | 92 ++++++++++++++++++++++++++++++++------ app/utils/Crypto.scala | 19 ++++++++ 2 files changed, 97 insertions(+), 14 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 931ab579..6ff1acaf 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -25,7 +25,7 @@ import scala.concurrent._ import javax.crypto.Mac import javax.xml.bind.DatatypeConverter -import java.math.BigInteger +import scala.math.BigInt import java.security.KeyStore import scala.concurrent.forkjoin._ import scala.collection.mutable.ArrayBuffer @@ -229,11 +229,11 @@ object Console | commands.Console - Generate and send votes for load testing and benchmarking | |SYNOPSIS - | commands.Console [gen_votes|add_khmacs] [--vote-count number] - | [--plaintexts path] [--ciphertexts path] [--ciphertexts-khmac path] - | [--voterid-len number] [--shared-secret password] [--port port] - | [--ssl boolean] [--host hostname] [--service-path path] - | [--election-ids path] + | commands.Console [gen_plaintexts|gen_votes|add_khmacs] + | [--vote-count number] [--plaintexts path] [--ciphertexts path] + | [--ciphertexts-khmac path] [--voterid-len number] + | [--shared-secret password] [--port port] [--ssl boolean] + | [--host hostname] [--service-path path] [--election-ids path] | |DESCRIPTION | In order to run this program, use: @@ -755,9 +755,9 @@ object Console ballot: PlaintextBallot, dto: ElectionDTO ) - : Array[Long] = + : Array[BigInt] = { - var array = new Array[Long](ballot.answers.length) + var array = new Array[BigInt](ballot.answers.length) if (dto.configuration.questions.length != ballot.answers.length) { val strBallot = Json.toJson(ballot).toString @@ -791,7 +791,7 @@ object Console } // Convert to long. Notice that the zeros on the left added by the last // chosen option won't be included - array(i) = strValue.toLong + array(i) = BigInt(strValue) } match { @@ -910,7 +910,7 @@ object Console }.seq // we need to generate vote_count encrypted ballots, fill the list with // random samples of the base list - val toEncrypt : Seq[(Long, Array[Long])] = + val toEncrypt : Seq[(Long, Array[BigInt])] = { val extraSize = vote_count - votes.length val extra = Array.fill(extraSize.toInt){ votes(scala.util.Random.nextInt(votes.length)) } @@ -930,7 +930,7 @@ object Console { throw new EncryptionError(s"${pks.length} != ${plaintext.length}") } - val encryptedVote = Crypto.encrypt(pks, plaintext) + val encryptedVote = Crypto.encryptBig(pks, plaintext) val line = generate_vote_line(electionId, encryptedVote) writePromise completeWith writer.write(line) } @@ -986,37 +986,71 @@ object Console promise.future } - private def generate_plaintexts_for_eid(eid: Long, numVotes: Long, dto: ElectionDTO) : String = + /** + * Generates 'numVotes' random plaintext ballots for election id 'eid', using + * election info 'dto'. + * Each plaintext ballot will fill a string line and it will have the format + * described by method @processPlaintextLine: + * eid||option1,option2|option1|| + */ + + private def generate_plaintexts_for_eid( + eid: Long, + numVotes: Long, + dto: ElectionDTO + ) + : String = { + // output variable var outText: String = "" + // election id, as a string val sEid = eid.toString + // iterate over all the plaintext ballots that need be generated for (i <- 0L until numVotes) { + // variable that contains one plaintext ballot var line: String = sEid + // iterate over all the questions this election has for (question <- dto.configuration.questions) { + // add questions separator line += "|" val diff = question.max - question.min + // buffer with all possible options for this question val optionsBuffer : scala.collection.mutable.ArrayBuffer[Long] = (0L until question.answers.size.toLong) .to[scala.collection.mutable.ArrayBuffer] val num_answers = question.min + scala.util.Random.nextInt(diff + 1) + // iterate over the number of answers to be generated for this question for (j <- 0 until num_answers) { + // add options separator if (0 != j) { line += "," } + // select a random, non-repeated option val option = optionsBuffer(scala.util.Random.nextInt(optionsBuffer.size)) + // remove this option for the next operation, to avoid options repetition optionsBuffer -= option + // add option to plaintext ballot line line += option.toString } } - outText += line + // add plaintext ballot line to output variable + outText += line + "\n" } outText } + /** + * Given a map election id -> election info, and the total number of + * plaintext ballots (vote_count) to be generated, it generates another map + * election id -> (votes, election info). + * We'll generate between a and a+1 plaintext ballots for each election id, + * where a = vote_count / num elections. + */ + private def generate_map_num_votes_dto( electionsInfoMap: scala.collection.mutable.HashMap[Long, ElectionDTO] ) @@ -1035,12 +1069,17 @@ object Console } // minimum number of votes per election val votesFloor = (vote_count.toFloat / numElections.toFloat).floor.toLong + // number of votes already "used" var votesSoFar: Long = 0 + // number of elections already used var electionsSoFar: Long = 0 + // the output variable val electionsVotesDtoMap = scala.collection.mutable.HashMap[Long, (Long, ElectionDTO)]() + // electionsInfoMap.foreach { case (eid, dto) => + // votes to be generated for this election val votesToGenThisEid = if (vote_count - votesSoFar > votesFloor*(numElections - electionsSoFar)) { votesFloor + 1 @@ -1049,6 +1088,7 @@ object Console { votesFloor } + // add element to map electionsVotesDtoMap += (eid -> (votesToGenThisEid, dto)) votesSoFar += votesToGenThisEid electionsSoFar += 1 @@ -1056,6 +1096,11 @@ object Console electionsVotesDtoMap } + /** + * Given a map of election id to election info, it generates --vote-count + * number of plaintext ballots and save them on --plaintexts + */ + private def generate_save_plaintexts( electionsInfoMap: scala.collection.mutable.HashMap[Long, ElectionDTO] ) @@ -1064,7 +1109,9 @@ object Console val promise = Promise[Unit]() Future { + // get number of votes per election id, and election info, in a map val electionsVotesDtoMap = generate_map_num_votes_dto(electionsInfoMap) + // writer to write into the output plaintexts file in a thread-safe way val writer = new FileWriter(Paths.get(plaintexts_path)) promise completeWith { Future.traverse (electionsVotesDtoMap) @@ -1073,7 +1120,9 @@ object Console val writePromise = Promise[Unit]() Future { + // generate plaintexts for this election val text = generate_plaintexts_for_eid(eid, numVotes, dto) + // write plaintexts for this election into output file writePromise completeWith writer.write(text) } .recover @@ -1094,16 +1143,24 @@ object Console promise.future } + /** + * Reads the --election-ids file, which consists of an election id per line, + * and returns a set of election ids + */ + private def getElectionsSet() : Future[scala.collection.immutable.Set[Long]] = { val promise = Promise[scala.collection.immutable.Set[Long]]() Future { + // convert string to path val eids_path = Paths.get(election_ids_path) + // read file, split it into string lines val electionsSet = io.Source.fromFile(election_ids_path) .getLines() + // convert each line to an integer representing the election id .toList.map { strEid => strEid.toLong @@ -1119,7 +1176,14 @@ object Console } /** - * + * Generates random plaintext ballots for a number of elections. + * It generates --vote-count number of plaintexts, for the elections mentioned + * on file --election-ids. The election-ids file should have an election id on + * each line. Election ids should not be repeated. The generated plaintexts + * will be saved on the file set by option --plaintexts (or its default + * value). The output format of the plaintexts file is compatible with the + * required input for command gen_votes. The number of votes should be at + * least as big as the number of elections. */ private def gen_plaintexts() : Future[Unit] = diff --git a/app/utils/Crypto.scala b/app/utils/Crypto.scala index f634cc4c..ef9a35e7 100644 --- a/app/utils/Crypto.scala +++ b/app/utils/Crypto.scala @@ -110,6 +110,13 @@ import java.math.BigInteger encryptEncoded(pks, encoded) } + def encryptBig(pks: Array[PublicKey], values: Array[BigInt]) = { + val encoded = pks.view.zipWithIndex.map( t => + encodeBig(t._1, values(t._2)) + ).toArray + encryptEncoded(pks, encoded) + } + /** return a random bigint between 0 and max - 1 */ def randomBigInt(max: BigInt) = { val rnd = new java.util.Random() @@ -133,6 +140,18 @@ import java.math.BigInteger } } + /** encode plaintext into subgroup defined by pk */ + def encodeBig(pk: PublicKey, value: BigInt) = { + val one = BigInt(1) + val m = value + one + val test = m.modPow(pk.q, pk.p) + if (test.equals(one)) { + m + } else { + -m.mod(pk.p) + } + } + /** encrypt an encoded plaintext */ def encryptEncoded(pks: Array[PublicKey], values: Array[BigInt]) = { var choices = Array[Choice]() From 4dd3e80c9c95dcaf4410dce3b632c155fa8df351 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 28 Mar 2017 15:25:06 +0200 Subject: [PATCH 53/68] change ballot encryption algo, iterate plaintexts lists, but not randomly --- app/commands/Console.scala | 53 ++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 6ff1acaf..004bcc00 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -908,38 +908,35 @@ object Console ballot : PlaintextBallot => (ballot.id, encodePlaintext( ballot, electionsInfoMap.get(ballot.id).get ) ) }.seq - // we need to generate vote_count encrypted ballots, fill the list with - // random samples of the base list - val toEncrypt : Seq[(Long, Array[BigInt])] = - { - val extraSize = vote_count - votes.length - val extra = Array.fill(extraSize.toInt){ votes(scala.util.Random.nextInt(votes.length)) } - votes ++ extra - } val writer = new FileWriter(Paths.get(ciphertexts_path)) - promise completeWith + // we need to generate vote_count encrypted ballots, iterate the + // plaintexts list as many times as we need + val futuresArray = + (0 until vote_count.toInt).toStream.map { - Future.traverse (toEncrypt) - { - case (electionId, plaintext) => - val writePromise = Promise[Unit]() - Future - { - val pks = pksMap.get(electionId).get - if (pks.length != plaintext.length) - { - throw new EncryptionError(s"${pks.length} != ${plaintext.length}") - } - val encryptedVote = Crypto.encryptBig(pks, plaintext) - val line = generate_vote_line(electionId, encryptedVote) - writePromise completeWith writer.write(line) - } - .recover + index => + val writePromise = Promise[Unit]() + Future + { + val (electionId, plaintext) = votes(index % votes.size) + val pks = pksMap.get(electionId).get + if (pks.length != plaintext.length) { - case error: Throwable => writePromise failure error + throw new EncryptionError(s"${pks.length} != ${plaintext.length}") } - writePromise.future - } + val encryptedVote = Crypto.encryptBig(pks, plaintext) + val line = generate_vote_line(electionId, encryptedVote) + writePromise completeWith writer.write(line) + } + .recover + { + case error: Throwable => writePromise failure error + } + writePromise.future + } + promise completeWith + { + Future.sequence (futuresArray) .map { _ => () From feee7cee98f2ddece9a84e1b05c31d223e4f0aaa Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 28 Mar 2017 15:37:05 +0200 Subject: [PATCH 54/68] fix comment --- app/commands/Console.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 004bcc00..f20f7832 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -777,7 +777,8 @@ object Console val answer = ballot.answers(i) for (j <- 0 until answer.options.length) { - // sum 1 as the encryption method can't encode value zero + // sum 1 as it would cause encoding problems, as zeros on the left are + // removed in the end val optionStrBase = ( answer.options(j) + 1 ).toString // Each chosen option needs to have the same length in number of // characters/digits, so we fill in with zeros on the left From 241740655913a6d5a8114d50f25aa84c5bcfc29a Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 28 Mar 2017 16:03:43 +0200 Subject: [PATCH 55/68] change sbt config so that runMain can exit properly --- app/commands/Console.scala | 1 + build.sbt | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/app/commands/Console.scala b/app/commands/Console.scala index f20f7832..41ffa16f 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -1258,6 +1258,7 @@ object Console else { showHelp() + System.exit(0) } } () diff --git a/build.sbt b/build.sbt index 16b9cc7b..037c7cc8 100644 --- a/build.sbt +++ b/build.sbt @@ -17,6 +17,10 @@ name := """agora-elections""" version := "1.0-SNAPSHOT" +fork in run := true + +trapExit in run := false + lazy val root = (project in file(".")).enablePlugins(PlayScala) scalaVersion := "2.11.1" From 7bcf5f85b8fab1af1cce5cd3ede1c966eb05cb16 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 28 Mar 2017 16:57:52 +0200 Subject: [PATCH 56/68] add test.jmx jmeter test file --- admin/test.jmx | 180 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 admin/test.jmx diff --git a/admin/test.jmx b/admin/test.jmx new file mode 100644 index 00000000..ead73f89 --- /dev/null +++ b/admin/test.jmx @@ -0,0 +1,180 @@ + + + + + + false + false + + + + + + + + continue + + false + ${loop_count} + + ${num_threads} + ${rampup_secs} + 1489135288000 + 1489135288000 + false + + + + + + + + + ${server_host} + ${server_port} + + + + + + 6 + + + + true + + + + false + ${ballot} + = + + + + + + + + https + + ${service_path}/api/election/${electionid}/voter/${userid} + POST + true + false + true + false + false + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + + + + + + + + false + standard + org.apache.jmeter.protocol.http.control.HC4CookieHandler + + + + ${csv_file_path} + + electionid,userid,ballot,khmac + | + false + false + false + shareMode.all + + + + + + Authorization + ${khmac} + + + Content-Type + application/json;charset=UTF-8 + + + + + + + + num_threads + ${__P(num_threads,10)} + = + + + loop_count + ${__P(loop_count,100)} + = + + + rampup_secs + ${__P(rampup_secs,10)} + = + + + csv_file_path + ${__P(csv_file_path,/home/agoraelections/jmeter/ciphertexts-khmac.csv)} + = + + + server_host + ${__P(server_host,localhost)} + = + + + server_port + ${__P(server_port,9000)} + = + + + service_path + ${__P(service_path,/elections)} + = + + + + + + + + From 3e301a8d3a4bf2ddb7c5cae730089d96a6316698 Mon Sep 17 00:00:00 2001 From: Findeton Date: Wed, 29 Mar 2017 14:02:37 +0200 Subject: [PATCH 57/68] add select_categories_1click --- app/models/Models.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/Models.scala b/app/models/Models.scala index e1d38cd3..082549ec 100644 --- a/app/models/Models.scala +++ b/app/models/Models.scala @@ -424,7 +424,8 @@ case class QuestionExtra( shuffle_all_options: Option[Boolean], shuffle_category_list: Option[Array[String]], show_points: Option[Boolean], - default_selected_option_ids: Option[Array[Int]]) + default_selected_option_ids: Option[Array[Int]], + select_categories_1click: Option[Boolean]) { def validate() = { From 69828ff80836542a30d568e79399015764c1969d Mon Sep 17 00:00:00 2001 From: Findeton Date: Fri, 31 Mar 2017 18:11:45 +0200 Subject: [PATCH 58/68] adding some tests --- app/commands/Console.scala | 356 +++++++++++++++++++------------------ build.sbt | 4 + conf/application.conf | 24 +++ conf/application.test.conf | 3 + test/BallotboxSpec.scala | 138 -------------- test/ElectionsSpec.scala | 116 ------------ test/MiscSpec.scala | 53 ------ 7 files changed, 217 insertions(+), 477 deletions(-) create mode 100644 conf/application.test.conf delete mode 100644 test/BallotboxSpec.scala delete mode 100644 test/ElectionsSpec.scala delete mode 100644 test/MiscSpec.scala diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 41ffa16f..9bd1f907 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -85,18 +85,16 @@ class FileWriter(path: Path) } } } -/** - * Command for admin purposes - * - * use runMain commands.Command from console - * or - * activator "run-main commands.Command " - * from CLI - */ -object Console -{ - //implicit val ec = ExecutionContext.fromExecutor(new ForkJoinPool(100)) - //implicit val slickExecutionContext = Akka.system.dispatchers.lookup("play.akka.actor.slick-context") + +trait ConsoleInterface { + def showHelp() : Unit + def parse_args(args: Array[String]) : Int + def gen_votes() : Future[Unit] + def add_khmacs() : Future[Unit] + def gen_plaintexts() : Future[Unit] +} + +class ConsoleImpl extends ConsoleInterface { // number of votes to create var vote_count : Long = 0 @@ -131,13 +129,48 @@ object Console val secureDefaultsWithSpecificOptions:com.ning.http.client.AsyncHttpClientConfig = builder.build() implicit val wsClient = new play.api.libs.ws.ning.NingWSClient(secureDefaultsWithSpecificOptions) + /** + * Generates random plaintext ballots for a number of elections. + * It generates --vote-count number of plaintexts, for the elections mentioned + * on file --election-ids. The election-ids file should have an election id on + * each line. Election ids should not be repeated. The generated plaintexts + * will be saved on the file set by option --plaintexts (or its default + * value). The output format of the plaintexts file is compatible with the + * required input for command gen_votes. The number of votes should be at + * least as big as the number of elections. + */ + def gen_plaintexts() + : Future[Unit] = + { + val promise = Promise[Unit]() + Future + { + promise completeWith + { + getElectionsSet() + .flatMap + { + electionsSet => get_election_info_all(electionsSet) + } + .flatMap + { + electionsInfoMap => generate_save_plaintexts(electionsInfoMap) + } + } + } + .recover + { + case error: Throwable => promise failure error + } + promise.future + } /** * Parse program arguments */ - private def parse_args(args: Array[String]) = + def parse_args(args: Array[String]) : Int = { - var arg_index = 0 + var arg_index : Int = 0 while (arg_index + 2 < args.length) { // number of ballots to create @@ -217,12 +250,111 @@ object Console args(arg_index + 1)) } } + arg_index + } + + /** + * Given a list of plaintexts, it generates their ciphertexts + */ + def gen_votes() : Future[Unit] = + { + val promise = Promise[Unit]() + Future + { + promise completeWith + { + parsePlaintexts() flatMap + { + case (ballotsList, electionsSet) => + val dumpPksFuture = dump_pks_elections(electionsSet) + get_election_info_all(electionsSet) flatMap + { + electionsInfoMap => + dumpPksFuture flatMap + { + pksDumped => + encryptBallots(ballotsList, electionsInfoMap) + } + } + } + } + } + .recover + { + case error: Throwable => promise failure error + } + promise.future + } + + /** + * Reads a file created by gen_votes and adds a khmac to each ballot + * + * The objective of this command is to prepare the ballots to be sent by + * jmeter. + * + * Encrypting the plaintext ballots takes some time, and it can be + * done at any moment. However, the khmacs will only be valid for a certain + * period of time, and therefore, it can be useful to first encrypt the + * ballots and only generate the khmacs just before sending the ballots to + * the server. + * + * The format of each line of the ciphertexts file that this command reads + * should be: + * + * electionId|voterId|ballot + * + * Notice that the separator character is '|' + * + * The format of each line of the ciphertexts_khmac file that this command + * creates is: + * + * electionId|voterId|ballot|khmac + * + * Notice that it simply adds the khmac. + */ + def add_khmacs() : Future[Unit] = + { + val promise = Promise[Unit]() + Future + { + // check that the file we want to read exists + if (Files.exists(Paths.get(ciphertexts_path))) + { + val now = Some(System.currentTimeMillis / 1000) + val writer = new FileWriter(Paths.get(ciphertexts_khmac_path)) + promise completeWith + { + Future.traverse (io.Source.fromFile(ciphertexts_path).getLines()) + { + file_line => + val array = file_line.split('|') + val eid = array(0).toLong + val voterId = array(1) + val khmac = get_khmac(voterId, "AuthEvent", eid, "vote", now) + writer.write(file_line + "|" + khmac + "\n") + } + .map + { + _ => () + } + } + } + else + { + throw new java.io.FileNotFoundException("tally does not exist") + } + } + .recover + { + case error: Throwable => promise failure error + } + promise.future } /** * Prints to the standard output how to use this program */ - private def showHelp() = + def showHelp() = { System.out.println( """NAME @@ -356,7 +488,7 @@ object Console * - A blank vote for an answer is represented with no characters * - For example '||' means a blank vote for the corresponding answer */ - private def processPlaintextLine( + def processPlaintextLine( line: String, lineNumber: Long) : PlaintextBallot = @@ -467,77 +599,13 @@ object Console ballot } - /** - * Reads a file created by gen_votes and adds a khmac to each ballot - * - * The objective of this command is to prepare the ballots to be sent by - * jmeter. - * - * Encrypting the plaintext ballots takes some time, and it can be - * done at any moment. However, the khmacs will only be valid for a certain - * period of time, and therefore, it can be useful to first encrypt the - * ballots and only generate the khmacs just before sending the ballots to - * the server. - * - * The format of each line of the ciphertexts file that this command reads - * should be: - * - * electionId|voterId|ballot - * - * Notice that the separator character is '|' - * - * The format of each line of the ciphertexts_khmac file that this command - * creates is: - * - * electionId|voterId|ballot|khmac - * - * Notice that it simply adds the khmac. - */ - private def add_khmacs() : Future[Unit] = - { - val promise = Promise[Unit]() - Future - { - // check that the file we want to read exists - if (Files.exists(Paths.get(ciphertexts_path))) - { - val now = Some(System.currentTimeMillis / 1000) - val writer = new FileWriter(Paths.get(ciphertexts_khmac_path)) - promise completeWith - { - Future.traverse (io.Source.fromFile(ciphertexts_path).getLines()) - { - file_line => - val array = file_line.split('|') - val eid = array(0).toLong - val voterId = array(1) - val khmac = get_khmac(voterId, "AuthEvent", eid, "vote", now) - writer.write(file_line + "|" + khmac + "\n") - } - .map - { - _ => () - } - } - } - else - { - throw new java.io.FileNotFoundException("tally does not exist") - } - } - .recover - { - case error: Throwable => promise failure error - } - promise.future - } /** * Opens the plaintexts file, and reads and parses each line. * See Console.processPlaintextLine() comments for more info on the format * of the plaintext file. */ - private def parsePlaintexts() + def parsePlaintexts() : Future[ (scala.collection.immutable.List[PlaintextBallot], scala.collection.immutable.Set[Long])] = @@ -586,7 +654,7 @@ object Console /** * Generate khmac */ - private def get_khmac( + def get_khmac( userId: String, objType: String, objId: Long, @@ -610,7 +678,7 @@ object Console * Makes an http request to agora-elections to get the election info. * Returns the parsed election info */ - private def get_election_info(electionId: Long) : Future[ElectionDTO] = + def get_election_info(electionId: Long) : Future[ElectionDTO] = { val promise = Promise[ElectionDTO] Future @@ -646,7 +714,7 @@ object Console * Given a set of election ids, it returns a map of the election ids and their * election info. */ - private def get_election_info_all( + def get_election_info_all( electionsSet: scala.collection.immutable.Set[Long] ) : Future[scala.collection.mutable.HashMap[Long, ElectionDTO]] = @@ -687,7 +755,7 @@ object Console * Makes an HTTP request to agora-elections to dump the public keys for a * given election id. */ - private def dump_pks(electionId: Long): Future[Unit] = + def dump_pks(electionId: Long): Future[Unit] = { val promise = Promise[Unit]() Future @@ -724,7 +792,7 @@ object Console * Given a set of election ids, it returns a future that will be completed * when all the public keys of all those elections have been dumped. */ - private def dump_pks_elections( + def dump_pks_elections( electionsSet: scala.collection.immutable.Set[Long] ) : Future[Unit] = @@ -751,7 +819,7 @@ object Console * Given a plaintext and the election info, it encodes each question's answers * into a number, ready to be encrypted */ - private def encodePlaintext( + def encodePlaintext( ballot: PlaintextBallot, dto: ElectionDTO ) @@ -810,7 +878,7 @@ object Console * Generate a random string with a given length and alphabet * Note: This is not truly random, it uses a pseudo-random generator */ - private def gen_rnd_str(len: Int, choices: String) : String = + def gen_rnd_str(len: Int, choices: String) : String = { (0 until len) map { @@ -821,7 +889,7 @@ object Console /** * Generate a random voter id */ - private def generate_voterid() : String = + def generate_voterid() : String = { gen_rnd_str(voterid_len, voterid_alphabet) } @@ -832,7 +900,7 @@ object Console * electionId|voterId|vote * Notice that it automatically adds a random voter id */ - private def generate_vote_line(id: Long, ballot: EncryptedVote) : String = + def generate_vote_line(id: Long, ballot: EncryptedVote) : String = { val vote = Json.toJson(ballot).toString val voteHash = Crypto.sha256(vote) @@ -846,7 +914,7 @@ object Console * Given a map of election ids and election info, it returns a map with the * election public keys */ - private def get_pks_map( + def get_pks_map( electionsInfoMap: scala.collection.mutable.HashMap[Long, ElectionDTO] ) : scala.collection.mutable.HashMap[Long, Array[PublicKey]] = @@ -893,7 +961,7 @@ object Console * file. * Read generate_vote_line for more info on the output file format. */ - private def encryptBallots( + def encryptBallots( ballotsList: scala.collection.immutable.List[PlaintextBallot], electionsInfoMap: scala.collection.mutable.HashMap[Long, ElectionDTO] ) @@ -951,39 +1019,6 @@ object Console promise.future } - /** - * Given a list of plaintexts, it generates their ciphertexts - */ - private def gen_votes() : Future[Unit] = - { - val promise = Promise[Unit]() - Future - { - promise completeWith - { - parsePlaintexts() flatMap - { - case (ballotsList, electionsSet) => - val dumpPksFuture = dump_pks_elections(electionsSet) - get_election_info_all(electionsSet) flatMap - { - electionsInfoMap => - dumpPksFuture flatMap - { - pksDumped => - encryptBallots(ballotsList, electionsInfoMap) - } - } - } - } - } - .recover - { - case error: Throwable => promise failure error - } - promise.future - } - /** * Generates 'numVotes' random plaintext ballots for election id 'eid', using * election info 'dto'. @@ -992,7 +1027,7 @@ object Console * eid||option1,option2|option1|| */ - private def generate_plaintexts_for_eid( + def generate_plaintexts_for_eid( eid: Long, numVotes: Long, dto: ElectionDTO @@ -1049,7 +1084,7 @@ object Console * where a = vote_count / num elections. */ - private def generate_map_num_votes_dto( + def generate_map_num_votes_dto( electionsInfoMap: scala.collection.mutable.HashMap[Long, ElectionDTO] ) : scala.collection.mutable.HashMap[Long, (Long, ElectionDTO)] @@ -1099,7 +1134,7 @@ object Console * number of plaintext ballots and save them on --plaintexts */ - private def generate_save_plaintexts( + def generate_save_plaintexts( electionsInfoMap: scala.collection.mutable.HashMap[Long, ElectionDTO] ) : Future[Unit] = @@ -1146,7 +1181,7 @@ object Console * and returns a set of election ids */ - private def getElectionsSet() + def getElectionsSet() : Future[scala.collection.immutable.Set[Long]] = { val promise = Promise[scala.collection.immutable.Set[Long]]() @@ -1173,55 +1208,36 @@ object Console promise.future } - /** - * Generates random plaintext ballots for a number of elections. - * It generates --vote-count number of plaintexts, for the elections mentioned - * on file --election-ids. The election-ids file should have an election id on - * each line. Election ids should not be repeated. The generated plaintexts - * will be saved on the file set by option --plaintexts (or its default - * value). The output format of the plaintexts file is compatible with the - * required input for command gen_votes. The number of votes should be at - * least as big as the number of elections. - */ - private def gen_plaintexts() - : Future[Unit] = - { - val promise = Promise[Unit]() - Future - { - promise completeWith - { - getElectionsSet() - .flatMap - { - electionsSet => get_election_info_all(electionsSet) - } - .flatMap - { - electionsInfoMap => generate_save_plaintexts(electionsInfoMap) - } - } - } - .recover - { - case error: Throwable => promise failure error - } - promise.future - } +} + +/** + * Command for admin purposes + * + * use runMain commands.Command from console + * or + * activator "run-main commands.Command " + * from CLI + */ +object Console +{ + //implicit val ec = ExecutionContext.fromExecutor(new ForkJoinPool(100)) + //implicit val slickExecutionContext = Akka.system.dispatchers.lookup("play.akka.actor.slick-context") + + val intf : ConsoleInterface = new ConsoleImpl() def main(args: Array[String]) : Unit = { if(0 == args.length) { - showHelp() + intf.showHelp() } else { - parse_args(args) + intf.parse_args(args) val command = args(0) if ("gen_votes" == command) { - gen_votes() onComplete + intf.gen_votes() onComplete { case Success(value) => println("gen_votes success") @@ -1233,7 +1249,7 @@ object Console } else if ("add_khmacs" == command) { - add_khmacs() onComplete + intf.add_khmacs() onComplete { case Success(value) => println("add_khmacs success") @@ -1245,7 +1261,7 @@ object Console } else if ("gen_plaintexts" == command) { - gen_plaintexts() onComplete + intf.gen_plaintexts() onComplete { case Success(value) => println("gen_plaintexts success") @@ -1257,7 +1273,7 @@ object Console } else { - showHelp() + intf.showHelp() System.exit(0) } } diff --git a/build.sbt b/build.sbt index 037c7cc8..f234d4eb 100644 --- a/build.sbt +++ b/build.sbt @@ -19,6 +19,10 @@ version := "1.0-SNAPSHOT" fork in run := true +fork in Test := true + +javaOptions in Test += "-Dconfig.file=conf/application.test.conf" + trapExit in run := false lazy val root = (project in file(".")).enablePlugins(PlayScala) diff --git a/conf/application.conf b/conf/application.conf index 8f989320..15352b6c 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -90,6 +90,30 @@ slick.default="models.*" application.global=global.Global +akka { + actor { + default-dispatcher { + # This will be used if you have set "executor = "fork-join-executor"" + fork-join-executor { + # Min number of threads to cap factor-based parallelism number to + parallelism-min = 8 + + # The parallelism factor is used to determine thread pool size using the + # following formula: ceil(available processors * factor). Resulting size + # is then bounded by the parallelism-min and parallelism-max values. + parallelism-factor = 3.0 + + # Max number of threads to cap factor-based parallelism number to + parallelism-max = 64 + + # Setting to "FIFO" to use queue like peeking mode which "poll" or "LIFO" to use stack + # like peeking mode which "pop". + task-peeking-mode = "FIFO" + } + } + } +} + # http://stackoverflow.com/questions/19780545/play-slick-with-securesocial-running-db-io-in-a-separate-thread-pool play { akka { diff --git a/conf/application.test.conf b/conf/application.test.conf new file mode 100644 index 00000000..40ae19bf --- /dev/null +++ b/conf/application.test.conf @@ -0,0 +1,3 @@ +ehcacheplugin=disabled +memcached.host="127.0.0.1:11211" +logger.memcached=WARN diff --git a/test/BallotboxSpec.scala b/test/BallotboxSpec.scala deleted file mode 100644 index 4ea5759d..00000000 --- a/test/BallotboxSpec.scala +++ /dev/null @@ -1,138 +0,0 @@ -/** - * This file is part of agora_elections. - * Copyright (C) 2014-2016 Agora Voting SL - - * agora_elections is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License. - - * agora_elections is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - - * You should have received a copy of the GNU Affero General Public License - * along with agora_elections. If not, see . -**/ -package test - -import controllers.routes -import utils.Response -import utils.JsonFormatters._ - -import org.specs2.mutable._ -import org.specs2.runner._ -import org.junit.runner._ - -import play.api.test._ -import play.api.test.Helpers._ -import play.api.libs.json.Json -import play.api.libs.json.Reads -import play.api.libs.json.JsSuccess - -import scala.util.Try -import scala.util.Failure -import scala.util.Success -import utils._ - -import scala.concurrent._ -import scala.concurrent.duration._ - -import models._ -import controllers._ -import play.api.db.slick.DB - -// FIXME add hash check test -@RunWith(classOf[JUnitRunner]) -class BallotboxSpec extends Specification with TestContexts with Response { - - val pks1020 = """[{"q": "24792774508736884642868649594982829646677044143456685966902090450389126928108831401260556520412635107010557472033959413182721740344201744439332485685961403243832055703485006331622597516714353334475003356107214415133930521931501335636267863542365051534250347372371067531454567272385185891163945756520887249904654258635354225185183883072436706698802915430665330310171817147030511296815138402638418197652072758525915640803066679883309656829521003317945389314422254112846989412579196000319352105328237736727287933765675623872956765501985588170384171812463052893055840132089533980513123557770728491280124996262883108653723", "p": "49585549017473769285737299189965659293354088286913371933804180900778253856217662802521113040825270214021114944067918826365443480688403488878664971371922806487664111406970012663245195033428706668950006712214428830267861043863002671272535727084730103068500694744742135062909134544770371782327891513041774499809308517270708450370367766144873413397605830861330660620343634294061022593630276805276836395304145517051831281606133359766619313659042006635890778628844508225693978825158392000638704210656475473454575867531351247745913531003971176340768343624926105786111680264179067961026247115541456982560249992525766217307447", "y": "13508098908027539440784872565963852589801134315490214338943176764836483642826139287845321515683717120380306277154055628326294507608619219418615557143216499594289645318240026038566886829367592524471392914903704781226769279964504637233468503819400275780389692993402172532069322059466650168544610528332541448517737477349241892135854753287819378120770750997761642638894924410113312379756439635652045600801662648509996855951194816645263112222062553013343235646559457633821952444151434434089012336057352241599491187136434213738494446172709662424227966766414134346388055354268602096835376151079526673696516280553593666107589", "g": "27257469383433468307851821232336029008797963446516266868278476598991619799718416119050669032044861635977216445034054414149795443466616532657735624478207460577590891079795564114912418442396707864995938563067755479563850474870766067031326511471051504594777928264027177308453446787478587442663554203039337902473879502917292403539820877956251471612701203572143972352943753791062696757791667318486190154610777475721752749567975013100844032853600120195534259802017090281900264646220781224136443700521419393245058421718455034330177739612895494553069450438317893406027741045575821283411891535713793639123109933196544017309147"}, {"q": "24792774508736884642868649594982829646677044143456685966902090450389126928108831401260556520412635107010557472033959413182721740344201744439332485685961403243832055703485006331622597516714353334475003356107214415133930521931501335636267863542365051534250347372371067531454567272385185891163945756520887249904654258635354225185183883072436706698802915430665330310171817147030511296815138402638418197652072758525915640803066679883309656829521003317945389314422254112846989412579196000319352105328237736727287933765675623872956765501985588170384171812463052893055840132089533980513123557770728491280124996262883108653723", "p": "49585549017473769285737299189965659293354088286913371933804180900778253856217662802521113040825270214021114944067918826365443480688403488878664971371922806487664111406970012663245195033428706668950006712214428830267861043863002671272535727084730103068500694744742135062909134544770371782327891513041774499809308517270708450370367766144873413397605830861330660620343634294061022593630276805276836395304145517051831281606133359766619313659042006635890778628844508225693978825158392000638704210656475473454575867531351247745913531003971176340768343624926105786111680264179067961026247115541456982560249992525766217307447", "y": "31297029205822268885109934561544608602226551039112423096402549637947957272731991384490183392409022722612106620608611126891966831755090546449989893634300800079970667005150302578667091006497824837222532245847901951555174396950008610097420473924217521376861890531635822604735534162871386978450992468493031003499877489309748655981166737937456509501504493651931555406164617066203407040999511236599399729264489639078620839280286466056838131704329524969392869702643746439900685050950772288960536343109589179956887355431627048809981098998158435159660104072149990254799572045096649738379551835785845681480411449382487219653409", "g": "27257469383433468307851821232336029008797963446516266868278476598991619799718416119050669032044861635977216445034054414149795443466616532657735624478207460577590891079795564114912418442396707864995938563067755479563850474870766067031326511471051504594777928264027177308453446787478587442663554203039337902473879502917292403539820877956251471612701203572143972352943753791062696757791667318486190154610777475721752749567975013100844032853600120195534259802017090281900264646220781224136443700521419393245058421718455034330177739612895494553069450438317893406027741045575821283411891535713793639123109933196544017309147"}, {"q": "24792774508736884642868649594982829646677044143456685966902090450389126928108831401260556520412635107010557472033959413182721740344201744439332485685961403243832055703485006331622597516714353334475003356107214415133930521931501335636267863542365051534250347372371067531454567272385185891163945756520887249904654258635354225185183883072436706698802915430665330310171817147030511296815138402638418197652072758525915640803066679883309656829521003317945389314422254112846989412579196000319352105328237736727287933765675623872956765501985588170384171812463052893055840132089533980513123557770728491280124996262883108653723", "p": "49585549017473769285737299189965659293354088286913371933804180900778253856217662802521113040825270214021114944067918826365443480688403488878664971371922806487664111406970012663245195033428706668950006712214428830267861043863002671272535727084730103068500694744742135062909134544770371782327891513041774499809308517270708450370367766144873413397605830861330660620343634294061022593630276805276836395304145517051831281606133359766619313659042006635890778628844508225693978825158392000638704210656475473454575867531351247745913531003971176340768343624926105786111680264179067961026247115541456982560249992525766217307447", "y": "28153039221047374758267050742258469188407714375035938721532943393478730897565403536508107271034470161491862917467036671234191594061901743326054047906010029686746918292239208850818204556361759410481858274118021155554294796620103237004461685393187877457724113104419448494181862271526366233662587437547960647309797858777370591388933819508188041869180927498771832754947955323240717585941583632417751853012192483838974392147015304758036865840584555098871789975977943566262267427057405787758933461905202016456744705696313896056687561449718558140672912358532178330002601533819297718750337836014564907646333413658482178522028", "g": "27257469383433468307851821232336029008797963446516266868278476598991619799718416119050669032044861635977216445034054414149795443466616532657735624478207460577590891079795564114912418442396707864995938563067755479563850474870766067031326511471051504594777928264027177308453446787478587442663554203039337902473879502917292403539820877956251471612701203572143972352943753791062696757791667318486190154610777475721752749567975013100844032853600120195534259802017090281900264646220781224136443700521419393245058421718455034330177739612895494553069450438317893406027741045575821283411891535713793639123109933196544017309147"}]""" - - "BallotboxApi" should { - - "reject bad auth" in new AppWithDbData() { - val voteJson = getVote(1, "1") - - val response = route(FakeRequest(POST, routes.BallotboxApi.vote(1, "1").url) - .withJsonBody(voteJson) - .withHeaders(("Authorization", "bogus")) - ).get - - status(response) must equalTo(FORBIDDEN) - } - - "allow casting votes" in new AppWithDbData() { - - DB.withSession { implicit session => - val cfg = TestData.config.validate[ElectionConfig].get - Elections.insert(Election(cfg.id, TestData.config.toString, Elections.REGISTERED, cfg.start_date, cfg.end_date, None, None, None, None)) - - // for validation to work we need to set the pk for the election manually (for election 1020) - Elections.setPublicKeys(1, pks1020) - Elections.updateState(1, Elections.STARTED) - } - - val voteJson = getVote(1, "1") - - val response = route(FakeRequest(POST, routes.BallotboxApi.vote(1, "1").url) - .withJsonBody(voteJson) - .withHeaders(("Authorization", getAuth("1", "AuthEvent", 1, "vote"))) - ).get - - status(response) must equalTo(OK) - } - - "allow dumping votes" in new AppWithDbData() { - val duration = (100, SECONDS) - - DB.withSession { implicit session => - val cfg = TestData.config.validate[ElectionConfig].get - Elections.insert(Election(cfg.id, TestData.config.toString, Elections.REGISTERED, cfg.start_date, cfg.end_date, None, None, None, None)) - - // for validation to work we need to set the pk for the election manually (for election 1020) - Elections.setPublicKeys(1, pks1020) - Elections.updateState(1, Elections.STARTED) - } - - val voteJson = getVote(1, "1") - - val response = route(FakeRequest(POST, routes.BallotboxApi.vote(1, "1").url) - .withJsonBody(voteJson) - .withHeaders(("Authorization", getAuth("1", "AuthEvent", 1, "vote"))) - ).get - - status(response) must equalTo(OK) - - val voteJson2 = getVote(1, "1") - - val response2 = route(FakeRequest(POST, routes.BallotboxApi.vote(1, "1").url) - .withJsonBody(voteJson) - .withHeaders(("Authorization", getAuth("1", "AuthEvent", 1, "vote"))) - ).get - - status(response2) must equalTo(OK) - - Await.result(BallotboxApi.dumpTheVotes(1), duration) - - val path = Datastore.getPath(1, Datastore.CIPHERTEXTS, false) - java.nio.file.Files.exists(path) must equalTo(true) - val lines = java.nio.file.Files.readAllLines(path, java.nio.charset.StandardCharsets.UTF_8) - lines.size() must equalTo(1) - } - } - - private def getVote(electionId: Long, voterId: String) = { - // this vote was cast with the 1020 election public keys, to test it that must be the public key for the election - val voteJson1020 = """{\"a\":\"encrypted-vote-v1\",\"choices\":[{\"alpha\":\"14268858776014931861458608313517070856036869918360554840257322856847542098845060563400252725408151504443689640611782069657557366076314034005926105640102753331283323683927731162586702452504414900483664232251706611593914945053069881853527205681629143320755824446175436453353447242777732548179187099820852306835018099740959374580596232340402230321382898301680181371244175479546234410117758889420419144117988991505324396563056696740570337369435889659133616195621252044920649614410101433256045437156662503299740904855635301136565548458580080122151343981193276667226335535758222123736164713319848692497837135317887590138216\",\"beta\":\"38129594905003909428559756787332613209800236112310345572458923168954311142426936463650020142811211367084243966765625305747152516201815205177928330617455237925660042831501439791160514162960248502061819095120746511150550371639863832578619685629121310674681673529648420418990628021761930770360516061129168052282673912203341582536041559384808650525048379036138240164375027301242348857687731117362577312375465367635984899278073552402864365569327443019073777602998458278982850323707111572658875136980943932951894871021137197698127874020371871548417036437233337397307383410152824383312669339892381058200681549884383129076542\"},{\"alpha\":\"29088883407482755631288867451785451145459495255834955876684619940876261866861340760858999039725793199905550736759985187515700467516831483521228814846648014792319339262966353721343458878923433601722023484967196736194772469411074500118618982321359719709087486677622562863624999853671283011342607539896094542966932006061927627229605414858487537314056224838699869740764547451782009309792152746470801601684065949481663009045824193830885321394443342165833468030820111626294257155562482713680587483972107747252941296163989153297751322456896284112940281013805173575569284220171249693183415381802710531603431441907039352428369\",\"beta\":\"25042815179618695804399500002996234931718703482174947163414199948094060354500739833473616751056318506697092943217039728639138855699136552221326440903435195552216050345002526004798097718966820003230130947417050070804262074732752078635123777039206371717171459023954179025763489859562265617250812641462941987431342878817086863319578864736091888605988579385216823599107029505389018275001418743060575057300470125877277971902121326888281683828672625546368348247467659441308860507497461312776066885011121445786913263909418699600048446540716846253389171367417854800831263585657410276737319332621050547020338863200099578606746\"},{\"alpha\":\"42644078246091347979735007022092136209266755972289295373847945355518063844794508247770301997311829803509491858959377290169416392933094695548277113982884810428692059003116484694088218651136681186364053257244508718686243890519274586485631097549467093086950436380220601170752603226409361311048915703911795506264605977756256960878477868559494037910571335091899861153023199553427077926815404735247696137765369774671022754223423751623939898282367893251661902903677815617891453340665519344393043081071965628116132951831972693750661288402737601333825816326430296687734033301458294444940914762156098396624096342388696403344169\",\"beta\":\"22653906121605281680685760322955912606342662560984462263105041951425368044384939563151441494324947953799361851688977731859585342952119023229195497416190950995909588068730932325137834072827152780506611971748001817559206644197698334584637312087753539461866414196771558237450764346145602329467524230159939566273604433592877530730940565056696934802987230354295291250712268352877639102818750186211679884915308518536418930340514484925808176090878163070755692202872961792817621486311374829949543333393943651376967739981859430459324679676416565990898155555954328276472556106626110239332707870259452352079504803770887074936359\"}],\"election_hash\":{\"a\":\"hash/sha256/value\",\"value\":\"\"},\"issue_date\":\"07/11/2014\",\"proofs\":[{\"challenge\":\"78001874093177159878093429504677748265743675106050781009101214808751744194883\",\"commitment\":\"19652325283417503212671394818084512364934039533193418160291185881640904729960917251875609186180633223191699485617506633218340019307742638726813206831220680708706253181945758931246817528832109616823058863695821341403412079244648960119342394320582881967191753762707975584805412395779487298045659163532558707077910201175721015792735149755455334433193370261714854706337622626605556913606021049650386574179482314401538549467702618760674254714829118975096426476928550691643757710807027349139785537307650181292863594132233848344194239452567352158916153934382874884399565736296461258440047483016141905564470609072527143068713\",\"response\":\"21709072417461931213953125316083694569255580351730652272998161309447698958876455601510820014923788994595219734250012459467528903588282529983131497289589870427004323150900845570710717578396594679196774114879934466887256147250456426197439728672159190620865504151769016906037439881699115129352174076600569736874206872942624876040481481104268987982038803475993731396703304944752509713002743701273201776476218413414649753658462837219505197844070987053775156984772157409602740115589049815593036724839811157510090374041020170578965777896378139929167009040565688888394695581600800734625846020646142264154743248773398850854327\"},{\"challenge\":\"52222863677320195509390108700954977702434493036788529471075256163916521317814\",\"commitment\":\"33409540474052906481284133472609676476184073506430290882051409016054952537098326592212657103493191167077907920287284757647414504416035218756956734267032847352200808722558657679290262608900393210370883519609075883079285988757278171284628924157833417597771347357651978932725560228239143981778188542977527938456247329448863732054816522977574886657373465874232910961563440519358017199330472216689546359142853017423024109251819405274786090602237346922946688678685565995497310428982822969575595145357915411697423489496880686383159064099364805813664486620333295368532055435782075438329698831639806192413682186598766171465258\",\"response\":\"19309735475877960945218363334252769260169120764531180612074712842957575209453606497928006665014028876503895892477468248455071694089176354094535485023564517923491041192889792789578916835101176257831094725537927822442181990575670937751769567017418969571931558759340621390692980223451180060010345126613210221647413725583499918253965351164731092958780285551115238265077083890342173885540198666674833001596721060506840931372804320143927733680856962715280307846716783905812674419874253277398533196722664214491510129158030985287669031868595351800855122812564668396670904624064865372170428700006599235536986299458575067504628\"},{\"challenge\":\"98612167410839376521210184409909984243962447292704239645546358359573426030292\",\"commitment\":\"41535988498679642176066812876417205373099865837917639390808483452010715668100835685806886348340086671454356264434376320916681242642175147018694416916543077039565905282427585863750545942908460262115171230343193432079245184821945805886998530161012361806045673862624824148394598215132603614334049231840703209742946542551518383978325315334921474327740409257647272143890479463352444669118349427307674670696441713399429237489664225058065323821185987413324354985100620302286475509793852259454915844156384966418477543311802937353389624043466865308891450688699030498903950598869492000510563282782204957773647081055599486048860\",\"response\":\"16674479995399418690845146487238515856893342614881736176308307212192388121380774827387230698374169053906553517846241330514230809966240808240083497717785882677472268225134856807096993392561052245125132704318318700911064776967848033910233890199778435208602974475572352297631654646177052215262113317082062371784386004448305257352076843537364556139740783127423851745444946040662794320592529073620551153678172522239378620512317668777473709467991392008553653577576223683531084689606010810110826962939986775580258467915862798447498836913801107495768024147381041889267743834925035028416298503620592844100211529297717528671856\"}]}""" - - val hash = Crypto.sha256(voteJson1020.replace("""\"""", """"""")) - val created = "2015-12-09T18:17:14.457" - - val newVoteJson = s"""{ - "vote": "$voteJson1020", - "vote_hash": "$hash" - }""" - - Json.parse(newVoteJson) - } -} \ No newline at end of file diff --git a/test/ElectionsSpec.scala b/test/ElectionsSpec.scala deleted file mode 100644 index 43dac66b..00000000 --- a/test/ElectionsSpec.scala +++ /dev/null @@ -1,116 +0,0 @@ -/** - * This file is part of agora_elections. - * Copyright (C) 2014-2016 Agora Voting SL - - * agora_elections is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License. - - * agora_elections is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - - * You should have received a copy of the GNU Affero General Public License - * along with agora_elections. If not, see . -**/ -package test - -import controllers.routes -import utils.Response -import utils.JsonFormatters._ - -import org.specs2.mutable._ -import org.specs2.runner._ -import org.junit.runner._ - -import play.api.test._ -import play.api.test.Helpers._ -import play.api.libs.json.Json -import play.api.libs.json.Reads -import play.api.libs.json.JsSuccess - -import scala.util.Try -import scala.util.Failure -import scala.util.Success - -import scala.concurrent._ - -import models._ -import play.api.db.slick.DB - - -@RunWith(classOf[JUnitRunner]) -class ElectionsSpec extends Specification with TestContexts with Response { - - "ElectionsApi" should { - - "reject bad auth" in new AppWithDbData() { - - val response = route(FakeRequest(POST, routes.ElectionsApi.register(1).url) - .withJsonBody(TestData.config) - .withHeaders(("Authorization", "bogus")) - ).get - - status(response) must equalTo(FORBIDDEN) - } - - "allow registering an retrieving an election" in new AppWithDbData() { - - // for this to work we need to set the pk for the election manually (for election 1020) - var response = route(FakeRequest(POST, routes.ElectionsApi.register(1).url) - .withJsonBody(TestData.config) - .withHeaders(("Authorization", getAuth("", "AuthEvent", 1, "edit"))) - ).get - - responseCheck(response, (r:Response[Int]) => r.payload > 0) - - response = route(FakeRequest(GET, routes.ElectionsApi.get(1).url) - .withJsonBody(TestData.config) - // .withHeaders(("Authorization", getAuth("", "election", 0, "admin"))) - ).get - - responseCheck(response, (r:Response[ElectionDTO]) => r.payload.configuration.title == "Votación de candidatos") - } - - "allow updating an election" in new AppWithDbData() { - - DB.withSession { implicit session => - val cfg = TestData.config.validate[ElectionConfig].get - Elections.insert(Election(cfg.id, TestData.config.toString, Elections.REGISTERED, cfg.start_date, cfg.end_date, None, None, None, None)) - } - - val response = route(FakeRequest(POST, routes.ElectionsApi.update(1).url) - .withJsonBody(TestData.config) - .withHeaders(("Authorization", getAuth("", "AuthEvent", 1, "edit"))) - ).get - - responseCheck(response, (r:Response[Int]) => r.payload > 0) - } - - "allow retrieving authority data" in new AppWithDbData() { - - val response = route(FakeRequest(GET, routes.ElectionsApi.getAuthorities.url) - .withHeaders(("Authorization", "bogus")) - ).get - - responseCheck(response, (r:Response[Map[String, AuthData]]) => r.payload.size == 2) - } - } - - def responseCheck[T: Reads](result: Future[play.api.mvc.Result], f: T => Boolean, code:Int = OK) = { - println(s">>> received '${contentAsString(result)}'") - - status(result) must equalTo(code) - contentType(result) must beSome.which(_ == "application/json") - - val parsed = Try(Json.parse(contentAsString(result))).map(_.validate[T]) - // force it to crash if parse errors - parsed.get - - parsed must beLike { - case Success(JsSuccess(response, _)) if f(response) => ok - case _ => ko - } - } -} \ No newline at end of file diff --git a/test/MiscSpec.scala b/test/MiscSpec.scala deleted file mode 100644 index 6c18aa29..00000000 --- a/test/MiscSpec.scala +++ /dev/null @@ -1,53 +0,0 @@ -/** - * This file is part of agora_elections. - * Copyright (C) 2014-2016 Agora Voting SL - - * agora_elections is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License. - - * agora_elections is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - - * You should have received a copy of the GNU Affero General Public License - * along with agora_elections. If not, see . -**/ -package test - -import org.specs2.mutable._ -import org.specs2.runner._ -import org.junit.runner._ - -import play.api.test._ -import play.api.test.Helpers._ -import play.api.libs.json.Json -import play.api.libs.json.Reads -import play.api.libs.json.JsSuccess - -import scala.concurrent._ -import scala.util.Try -import scala.util.Failure -import scala.util.Success - -import controllers.routes -import utils._ - -// FIXME add legendre tests -/** */ -@RunWith(classOf[JUnitRunner]) -class MiscSpec extends Specification { - - "Misc utils" should { - - "allow correct hmac calculation" in { - Crypto.hmac("pajarraco", "forrarme:1417267291") must beEqualTo("d5c4e961a496559b0a19039bc9cb62a3d9b63b38c54eb14286ca54364d21841e") - } - - /** this does not work as the dumped votes vary each test, you can test sha256 using echo "blabla" | sha512sum to verify */ - /* "allow correct hashing of file" in new WithApplication() { - Datastore.hashVotes(1) must beEqualTo("cf53a19dc191e3af395e463b8664578d4daa8770c4e98d0ab863ffdd47784f18") - } */ - } -} \ No newline at end of file From 825c47a9fbd78f163c0aadebb9ca8ed3ef895254 Mon Sep 17 00:00:00 2001 From: Findeton Date: Mon, 3 Apr 2017 19:04:02 +0200 Subject: [PATCH 59/68] add ConsoleSpec Console tests --- app/commands/Console.scala | 15 +- app/models/Models.scala | 10 + test/BallotboxSpec.scala | 138 ++++++++++ test/ConsoleSpec.scala | 499 +++++++++++++++++++++++++++++++++++++ test/ElectionsSpec.scala | 116 +++++++++ test/MiscSpec.scala | 53 ++++ 6 files changed, 819 insertions(+), 12 deletions(-) create mode 100644 test/BallotboxSpec.scala create mode 100644 test/ConsoleSpec.scala create mode 100644 test/ElectionsSpec.scala create mode 100644 test/MiscSpec.scala diff --git a/app/commands/Console.scala b/app/commands/Console.scala index 9bd1f907..a9fc8ee3 100644 --- a/app/commands/Console.scala +++ b/app/commands/Console.scala @@ -59,16 +59,6 @@ case class EncodePlaintextError(message: String) extends Exception(message) case class EncryptionError(message: String) extends Exception(message) case class ElectionIdsFileError(message: String) extends Exception(message) -/** - * This object contains the states required for reading a plaintext ballot - * It's used on Console.processPlaintextLine - */ -object PlaintextBallot -{ - val ID = 0 // reading election ID - val ANSWER = 1 // reading answers -} - /** * A simple class to write lines to a single file, in a multi-threading safe way */ @@ -1109,9 +1099,10 @@ class ConsoleImpl extends ConsoleInterface { // the output variable val electionsVotesDtoMap = scala.collection.mutable.HashMap[Long, (Long, ElectionDTO)]() // - electionsInfoMap.foreach + electionsInfoMap.keys.toList.sorted foreach { - case (eid, dto) => + case eid => + val dto = electionsInfoMap.get(eid).get // votes to be generated for this election val votesToGenThisEid = if (vote_count - votesSoFar > votesFloor*(numElections - electionsSoFar)) { diff --git a/app/models/Models.scala b/app/models/Models.scala index dd242746..e787bf8f 100644 --- a/app/models/Models.scala +++ b/app/models/Models.scala @@ -649,3 +649,13 @@ case class AuthData(name: Option[String], description: Option[String], url: Opti case class PlaintextAnswer(options: Array[Long] = Array[Long]()) // id is the election ID case class PlaintextBallot(id: Long = -1, answers: Array[PlaintextAnswer] = Array[PlaintextAnswer]()) + +/** + * This object contains the states required for reading a plaintext ballot + * It's used on Console.processPlaintextLine + */ +object PlaintextBallot +{ + val ID = 0 // reading election ID + val ANSWER = 1 // reading answers +} \ No newline at end of file diff --git a/test/BallotboxSpec.scala b/test/BallotboxSpec.scala new file mode 100644 index 00000000..4ea5759d --- /dev/null +++ b/test/BallotboxSpec.scala @@ -0,0 +1,138 @@ +/** + * This file is part of agora_elections. + * Copyright (C) 2014-2016 Agora Voting SL + + * agora_elections is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License. + + * agora_elections is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + + * You should have received a copy of the GNU Affero General Public License + * along with agora_elections. If not, see . +**/ +package test + +import controllers.routes +import utils.Response +import utils.JsonFormatters._ + +import org.specs2.mutable._ +import org.specs2.runner._ +import org.junit.runner._ + +import play.api.test._ +import play.api.test.Helpers._ +import play.api.libs.json.Json +import play.api.libs.json.Reads +import play.api.libs.json.JsSuccess + +import scala.util.Try +import scala.util.Failure +import scala.util.Success +import utils._ + +import scala.concurrent._ +import scala.concurrent.duration._ + +import models._ +import controllers._ +import play.api.db.slick.DB + +// FIXME add hash check test +@RunWith(classOf[JUnitRunner]) +class BallotboxSpec extends Specification with TestContexts with Response { + + val pks1020 = """[{"q": "24792774508736884642868649594982829646677044143456685966902090450389126928108831401260556520412635107010557472033959413182721740344201744439332485685961403243832055703485006331622597516714353334475003356107214415133930521931501335636267863542365051534250347372371067531454567272385185891163945756520887249904654258635354225185183883072436706698802915430665330310171817147030511296815138402638418197652072758525915640803066679883309656829521003317945389314422254112846989412579196000319352105328237736727287933765675623872956765501985588170384171812463052893055840132089533980513123557770728491280124996262883108653723", "p": "49585549017473769285737299189965659293354088286913371933804180900778253856217662802521113040825270214021114944067918826365443480688403488878664971371922806487664111406970012663245195033428706668950006712214428830267861043863002671272535727084730103068500694744742135062909134544770371782327891513041774499809308517270708450370367766144873413397605830861330660620343634294061022593630276805276836395304145517051831281606133359766619313659042006635890778628844508225693978825158392000638704210656475473454575867531351247745913531003971176340768343624926105786111680264179067961026247115541456982560249992525766217307447", "y": "13508098908027539440784872565963852589801134315490214338943176764836483642826139287845321515683717120380306277154055628326294507608619219418615557143216499594289645318240026038566886829367592524471392914903704781226769279964504637233468503819400275780389692993402172532069322059466650168544610528332541448517737477349241892135854753287819378120770750997761642638894924410113312379756439635652045600801662648509996855951194816645263112222062553013343235646559457633821952444151434434089012336057352241599491187136434213738494446172709662424227966766414134346388055354268602096835376151079526673696516280553593666107589", "g": "27257469383433468307851821232336029008797963446516266868278476598991619799718416119050669032044861635977216445034054414149795443466616532657735624478207460577590891079795564114912418442396707864995938563067755479563850474870766067031326511471051504594777928264027177308453446787478587442663554203039337902473879502917292403539820877956251471612701203572143972352943753791062696757791667318486190154610777475721752749567975013100844032853600120195534259802017090281900264646220781224136443700521419393245058421718455034330177739612895494553069450438317893406027741045575821283411891535713793639123109933196544017309147"}, {"q": "24792774508736884642868649594982829646677044143456685966902090450389126928108831401260556520412635107010557472033959413182721740344201744439332485685961403243832055703485006331622597516714353334475003356107214415133930521931501335636267863542365051534250347372371067531454567272385185891163945756520887249904654258635354225185183883072436706698802915430665330310171817147030511296815138402638418197652072758525915640803066679883309656829521003317945389314422254112846989412579196000319352105328237736727287933765675623872956765501985588170384171812463052893055840132089533980513123557770728491280124996262883108653723", "p": "49585549017473769285737299189965659293354088286913371933804180900778253856217662802521113040825270214021114944067918826365443480688403488878664971371922806487664111406970012663245195033428706668950006712214428830267861043863002671272535727084730103068500694744742135062909134544770371782327891513041774499809308517270708450370367766144873413397605830861330660620343634294061022593630276805276836395304145517051831281606133359766619313659042006635890778628844508225693978825158392000638704210656475473454575867531351247745913531003971176340768343624926105786111680264179067961026247115541456982560249992525766217307447", "y": "31297029205822268885109934561544608602226551039112423096402549637947957272731991384490183392409022722612106620608611126891966831755090546449989893634300800079970667005150302578667091006497824837222532245847901951555174396950008610097420473924217521376861890531635822604735534162871386978450992468493031003499877489309748655981166737937456509501504493651931555406164617066203407040999511236599399729264489639078620839280286466056838131704329524969392869702643746439900685050950772288960536343109589179956887355431627048809981098998158435159660104072149990254799572045096649738379551835785845681480411449382487219653409", "g": "27257469383433468307851821232336029008797963446516266868278476598991619799718416119050669032044861635977216445034054414149795443466616532657735624478207460577590891079795564114912418442396707864995938563067755479563850474870766067031326511471051504594777928264027177308453446787478587442663554203039337902473879502917292403539820877956251471612701203572143972352943753791062696757791667318486190154610777475721752749567975013100844032853600120195534259802017090281900264646220781224136443700521419393245058421718455034330177739612895494553069450438317893406027741045575821283411891535713793639123109933196544017309147"}, {"q": "24792774508736884642868649594982829646677044143456685966902090450389126928108831401260556520412635107010557472033959413182721740344201744439332485685961403243832055703485006331622597516714353334475003356107214415133930521931501335636267863542365051534250347372371067531454567272385185891163945756520887249904654258635354225185183883072436706698802915430665330310171817147030511296815138402638418197652072758525915640803066679883309656829521003317945389314422254112846989412579196000319352105328237736727287933765675623872956765501985588170384171812463052893055840132089533980513123557770728491280124996262883108653723", "p": "49585549017473769285737299189965659293354088286913371933804180900778253856217662802521113040825270214021114944067918826365443480688403488878664971371922806487664111406970012663245195033428706668950006712214428830267861043863002671272535727084730103068500694744742135062909134544770371782327891513041774499809308517270708450370367766144873413397605830861330660620343634294061022593630276805276836395304145517051831281606133359766619313659042006635890778628844508225693978825158392000638704210656475473454575867531351247745913531003971176340768343624926105786111680264179067961026247115541456982560249992525766217307447", "y": "28153039221047374758267050742258469188407714375035938721532943393478730897565403536508107271034470161491862917467036671234191594061901743326054047906010029686746918292239208850818204556361759410481858274118021155554294796620103237004461685393187877457724113104419448494181862271526366233662587437547960647309797858777370591388933819508188041869180927498771832754947955323240717585941583632417751853012192483838974392147015304758036865840584555098871789975977943566262267427057405787758933461905202016456744705696313896056687561449718558140672912358532178330002601533819297718750337836014564907646333413658482178522028", "g": "27257469383433468307851821232336029008797963446516266868278476598991619799718416119050669032044861635977216445034054414149795443466616532657735624478207460577590891079795564114912418442396707864995938563067755479563850474870766067031326511471051504594777928264027177308453446787478587442663554203039337902473879502917292403539820877956251471612701203572143972352943753791062696757791667318486190154610777475721752749567975013100844032853600120195534259802017090281900264646220781224136443700521419393245058421718455034330177739612895494553069450438317893406027741045575821283411891535713793639123109933196544017309147"}]""" + + "BallotboxApi" should { + + "reject bad auth" in new AppWithDbData() { + val voteJson = getVote(1, "1") + + val response = route(FakeRequest(POST, routes.BallotboxApi.vote(1, "1").url) + .withJsonBody(voteJson) + .withHeaders(("Authorization", "bogus")) + ).get + + status(response) must equalTo(FORBIDDEN) + } + + "allow casting votes" in new AppWithDbData() { + + DB.withSession { implicit session => + val cfg = TestData.config.validate[ElectionConfig].get + Elections.insert(Election(cfg.id, TestData.config.toString, Elections.REGISTERED, cfg.start_date, cfg.end_date, None, None, None, None)) + + // for validation to work we need to set the pk for the election manually (for election 1020) + Elections.setPublicKeys(1, pks1020) + Elections.updateState(1, Elections.STARTED) + } + + val voteJson = getVote(1, "1") + + val response = route(FakeRequest(POST, routes.BallotboxApi.vote(1, "1").url) + .withJsonBody(voteJson) + .withHeaders(("Authorization", getAuth("1", "AuthEvent", 1, "vote"))) + ).get + + status(response) must equalTo(OK) + } + + "allow dumping votes" in new AppWithDbData() { + val duration = (100, SECONDS) + + DB.withSession { implicit session => + val cfg = TestData.config.validate[ElectionConfig].get + Elections.insert(Election(cfg.id, TestData.config.toString, Elections.REGISTERED, cfg.start_date, cfg.end_date, None, None, None, None)) + + // for validation to work we need to set the pk for the election manually (for election 1020) + Elections.setPublicKeys(1, pks1020) + Elections.updateState(1, Elections.STARTED) + } + + val voteJson = getVote(1, "1") + + val response = route(FakeRequest(POST, routes.BallotboxApi.vote(1, "1").url) + .withJsonBody(voteJson) + .withHeaders(("Authorization", getAuth("1", "AuthEvent", 1, "vote"))) + ).get + + status(response) must equalTo(OK) + + val voteJson2 = getVote(1, "1") + + val response2 = route(FakeRequest(POST, routes.BallotboxApi.vote(1, "1").url) + .withJsonBody(voteJson) + .withHeaders(("Authorization", getAuth("1", "AuthEvent", 1, "vote"))) + ).get + + status(response2) must equalTo(OK) + + Await.result(BallotboxApi.dumpTheVotes(1), duration) + + val path = Datastore.getPath(1, Datastore.CIPHERTEXTS, false) + java.nio.file.Files.exists(path) must equalTo(true) + val lines = java.nio.file.Files.readAllLines(path, java.nio.charset.StandardCharsets.UTF_8) + lines.size() must equalTo(1) + } + } + + private def getVote(electionId: Long, voterId: String) = { + // this vote was cast with the 1020 election public keys, to test it that must be the public key for the election + val voteJson1020 = """{\"a\":\"encrypted-vote-v1\",\"choices\":[{\"alpha\":\"14268858776014931861458608313517070856036869918360554840257322856847542098845060563400252725408151504443689640611782069657557366076314034005926105640102753331283323683927731162586702452504414900483664232251706611593914945053069881853527205681629143320755824446175436453353447242777732548179187099820852306835018099740959374580596232340402230321382898301680181371244175479546234410117758889420419144117988991505324396563056696740570337369435889659133616195621252044920649614410101433256045437156662503299740904855635301136565548458580080122151343981193276667226335535758222123736164713319848692497837135317887590138216\",\"beta\":\"38129594905003909428559756787332613209800236112310345572458923168954311142426936463650020142811211367084243966765625305747152516201815205177928330617455237925660042831501439791160514162960248502061819095120746511150550371639863832578619685629121310674681673529648420418990628021761930770360516061129168052282673912203341582536041559384808650525048379036138240164375027301242348857687731117362577312375465367635984899278073552402864365569327443019073777602998458278982850323707111572658875136980943932951894871021137197698127874020371871548417036437233337397307383410152824383312669339892381058200681549884383129076542\"},{\"alpha\":\"29088883407482755631288867451785451145459495255834955876684619940876261866861340760858999039725793199905550736759985187515700467516831483521228814846648014792319339262966353721343458878923433601722023484967196736194772469411074500118618982321359719709087486677622562863624999853671283011342607539896094542966932006061927627229605414858487537314056224838699869740764547451782009309792152746470801601684065949481663009045824193830885321394443342165833468030820111626294257155562482713680587483972107747252941296163989153297751322456896284112940281013805173575569284220171249693183415381802710531603431441907039352428369\",\"beta\":\"25042815179618695804399500002996234931718703482174947163414199948094060354500739833473616751056318506697092943217039728639138855699136552221326440903435195552216050345002526004798097718966820003230130947417050070804262074732752078635123777039206371717171459023954179025763489859562265617250812641462941987431342878817086863319578864736091888605988579385216823599107029505389018275001418743060575057300470125877277971902121326888281683828672625546368348247467659441308860507497461312776066885011121445786913263909418699600048446540716846253389171367417854800831263585657410276737319332621050547020338863200099578606746\"},{\"alpha\":\"42644078246091347979735007022092136209266755972289295373847945355518063844794508247770301997311829803509491858959377290169416392933094695548277113982884810428692059003116484694088218651136681186364053257244508718686243890519274586485631097549467093086950436380220601170752603226409361311048915703911795506264605977756256960878477868559494037910571335091899861153023199553427077926815404735247696137765369774671022754223423751623939898282367893251661902903677815617891453340665519344393043081071965628116132951831972693750661288402737601333825816326430296687734033301458294444940914762156098396624096342388696403344169\",\"beta\":\"22653906121605281680685760322955912606342662560984462263105041951425368044384939563151441494324947953799361851688977731859585342952119023229195497416190950995909588068730932325137834072827152780506611971748001817559206644197698334584637312087753539461866414196771558237450764346145602329467524230159939566273604433592877530730940565056696934802987230354295291250712268352877639102818750186211679884915308518536418930340514484925808176090878163070755692202872961792817621486311374829949543333393943651376967739981859430459324679676416565990898155555954328276472556106626110239332707870259452352079504803770887074936359\"}],\"election_hash\":{\"a\":\"hash/sha256/value\",\"value\":\"\"},\"issue_date\":\"07/11/2014\",\"proofs\":[{\"challenge\":\"78001874093177159878093429504677748265743675106050781009101214808751744194883\",\"commitment\":\"19652325283417503212671394818084512364934039533193418160291185881640904729960917251875609186180633223191699485617506633218340019307742638726813206831220680708706253181945758931246817528832109616823058863695821341403412079244648960119342394320582881967191753762707975584805412395779487298045659163532558707077910201175721015792735149755455334433193370261714854706337622626605556913606021049650386574179482314401538549467702618760674254714829118975096426476928550691643757710807027349139785537307650181292863594132233848344194239452567352158916153934382874884399565736296461258440047483016141905564470609072527143068713\",\"response\":\"21709072417461931213953125316083694569255580351730652272998161309447698958876455601510820014923788994595219734250012459467528903588282529983131497289589870427004323150900845570710717578396594679196774114879934466887256147250456426197439728672159190620865504151769016906037439881699115129352174076600569736874206872942624876040481481104268987982038803475993731396703304944752509713002743701273201776476218413414649753658462837219505197844070987053775156984772157409602740115589049815593036724839811157510090374041020170578965777896378139929167009040565688888394695581600800734625846020646142264154743248773398850854327\"},{\"challenge\":\"52222863677320195509390108700954977702434493036788529471075256163916521317814\",\"commitment\":\"33409540474052906481284133472609676476184073506430290882051409016054952537098326592212657103493191167077907920287284757647414504416035218756956734267032847352200808722558657679290262608900393210370883519609075883079285988757278171284628924157833417597771347357651978932725560228239143981778188542977527938456247329448863732054816522977574886657373465874232910961563440519358017199330472216689546359142853017423024109251819405274786090602237346922946688678685565995497310428982822969575595145357915411697423489496880686383159064099364805813664486620333295368532055435782075438329698831639806192413682186598766171465258\",\"response\":\"19309735475877960945218363334252769260169120764531180612074712842957575209453606497928006665014028876503895892477468248455071694089176354094535485023564517923491041192889792789578916835101176257831094725537927822442181990575670937751769567017418969571931558759340621390692980223451180060010345126613210221647413725583499918253965351164731092958780285551115238265077083890342173885540198666674833001596721060506840931372804320143927733680856962715280307846716783905812674419874253277398533196722664214491510129158030985287669031868595351800855122812564668396670904624064865372170428700006599235536986299458575067504628\"},{\"challenge\":\"98612167410839376521210184409909984243962447292704239645546358359573426030292\",\"commitment\":\"41535988498679642176066812876417205373099865837917639390808483452010715668100835685806886348340086671454356264434376320916681242642175147018694416916543077039565905282427585863750545942908460262115171230343193432079245184821945805886998530161012361806045673862624824148394598215132603614334049231840703209742946542551518383978325315334921474327740409257647272143890479463352444669118349427307674670696441713399429237489664225058065323821185987413324354985100620302286475509793852259454915844156384966418477543311802937353389624043466865308891450688699030498903950598869492000510563282782204957773647081055599486048860\",\"response\":\"16674479995399418690845146487238515856893342614881736176308307212192388121380774827387230698374169053906553517846241330514230809966240808240083497717785882677472268225134856807096993392561052245125132704318318700911064776967848033910233890199778435208602974475572352297631654646177052215262113317082062371784386004448305257352076843537364556139740783127423851745444946040662794320592529073620551153678172522239378620512317668777473709467991392008553653577576223683531084689606010810110826962939986775580258467915862798447498836913801107495768024147381041889267743834925035028416298503620592844100211529297717528671856\"}]}""" + + val hash = Crypto.sha256(voteJson1020.replace("""\"""", """"""")) + val created = "2015-12-09T18:17:14.457" + + val newVoteJson = s"""{ + "vote": "$voteJson1020", + "vote_hash": "$hash" + }""" + + Json.parse(newVoteJson) + } +} \ No newline at end of file diff --git a/test/ConsoleSpec.scala b/test/ConsoleSpec.scala new file mode 100644 index 00000000..f0170668 --- /dev/null +++ b/test/ConsoleSpec.scala @@ -0,0 +1,499 @@ +/** + * This file is part of agora_elections. + * Copyright (C) 2017 Agora Voting SL + + * agora_elections is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License. + + * agora_elections is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + + * You should have received a copy of the GNU Affero General Public License + * along with agora_elections. If not, see . +**/ + +package test + +import org.specs2.mutable._ +import org.specs2.runner._ +import org.junit.runner._ + +import play.api.libs.json._ +import play.api.mvc.Action +import play.api.mvc.Results._ +import play.api.test._ +import commands._ +import models._ +import utils.JsonFormatters._ +import utils.Crypto +import scala.concurrent._ +import scala.concurrent.duration._ +import scala.util._ +import play.api.cache.Cache +import java.sql.Timestamp + + +@RunWith(classOf[JUnitRunner]) +class ConsoleSpec extends Specification with TestContexts +{ + val timeoutDuration = Duration(20, SECONDS) + val dtoStr = +"""{ + "id": 42, + "configuration": { + "id": 42, + "layout": "simple", + "director": "auth1", + "authorities": [ + "auth2" + ], + "title": "New election", + "description": "", + "questions": [ + { + "description": "", + "layout": "accordion", + "max": 2, + "min": 1, + "num_winners": 1, + "title": "New question 0", + "tally_type": "plurality-at-large", + "answer_total_votes_percentage": "over-total-valid-votes", + "answers": [ + { + "id": 0, + "category": "", + "details": "this is a long descriptionthis is a long descriptionthis is a long descriptionthis is a long description", + "sort_order": 0, + "urls": [], + "text": "ab" + }, + { + "id": 1, + "category": "", + "details": "this is a long descriptionthis is a long descriptionthis is a long descriptionthis is a long descriptionthis is a long description", + "sort_order": 1, + "urls": [], + "text": "cd" + } + ], + "extra_options": { + "shuffle_categories": true, + "shuffle_all_options": true, + "shuffle_category_list": [], + "show_points": false + } + } + ], + "start_date": "2015-01-27T16:00:00.001", + "end_date": "2015-01-27T16:00:00.001", + "presentation": { + "share_text": [ + { + "network": "Twitter", + "button_text": "", + "social_message": "I have just voted in election https://go.nvotes.com, you can too! #nvotes" + } + ], + "theme": "default", + "urls": [], + "theme_css": "" + }, + "real": false, + "extra_data": "{}", + "virtual": false, + "virtualSubelections": [], + "logo_url": "" + }, + "state": "created", + "startDate": "2015-01-27T16:00:00.001", + "endDate": "2015-01-27T16:00:00.001", + "pks": "[{\"q\":\"24792774508736884642868649594982829646677044143456685966902090450389126928108831401260556520412635107010557472033959413182721740344201744439332485685961403243832055703485006331622597516714353334475003356107214415133930521931501335636267863542365051534250347372371067531454567272385185891163945756520887249904654258635354225185183883072436706698802915430665330310171817147030511296815138402638418197652072758525915640803066679883309656829521003317945389314422254112846989412579196000319352105328237736727287933765675623872956765501985588170384171812463052893055840132089533980513123557770728491280124996262883108653723\",\"p\":\"49585549017473769285737299189965659293354088286913371933804180900778253856217662802521113040825270214021114944067918826365443480688403488878664971371922806487664111406970012663245195033428706668950006712214428830267861043863002671272535727084730103068500694744742135062909134544770371782327891513041774499809308517270708450370367766144873413397605830861330660620343634294061022593630276805276836395304145517051831281606133359766619313659042006635890778628844508225693978825158392000638704210656475473454575867531351247745913531003971176340768343624926105786111680264179067961026247115541456982560249992525766217307447\",\"y\":\"36546166940599313416168254217744697729911998485163232812421755063790894563031509201724580104715842554805784888514324341999107803878734424284146217463747343381092445557201708138031676925435834811853723665739792579746520854169833006814210270893803218238156148054483205249064331467974912676179467375074903261138777633846359449221132575280279543204461738096325062062234143265942024786615427875783393149444592255523696808235191395788053253235712555210828055008777307102133087648043115255896201502913954041965429673426222803318582002444673454964248205507662853386137487981094735018694462667314177271930377544606339760319819\",\"g\":\"27257469383433468307851821232336029008797963446516266868278476598991619799718416119050669032044861635977216445034054414149795443466616532657735624478207460577590891079795564114912418442396707864995938563067755479563850474870766067031326511471051504594777928264027177308453446787478587442663554203039337902473879502917292403539820877956251471612701203572143972352943753791062696757791667318486190154610777475721752749567975013100844032853600120195534259802017090281900264646220781224136443700521419393245058421718455034330177739612895494553069450438317893406027741045575821283411891535713793639123109933196544017309147\"}]", + "real": false, + "virtual": false, + "logo_url": "" +}""" + val dto : ElectionDTO = Json.parse(dtoStr).validate[ElectionDTO].get + + "parse_args" should + { + "read nothing with only one argument" in + { + val console = new ConsoleImpl() + val array: Array[String] = Array("console") + console.parse_args(array) must be equalTo(0) + } + + "read nothing with only two arguments" in + { + val console = new ConsoleImpl() + val array: Array[String] = Array("console", "whatever") + console.parse_args(array) must be equalTo(0) + } + + "throw IllegalArgumentException with wrong arguments" in + { + val console = new ConsoleImpl() + val array: Array[String] = Array("console", "--whatever", "value") + console.parse_args(array) must throwAn[java.lang.IllegalArgumentException] + } + + "throw IllegalArgumentException with invalid port" in + { + val console = new ConsoleImpl() + var array: Array[String] = Array("gen_votes", "--port", "value") + console.parse_args(array) must throwAn[java.lang.IllegalArgumentException] + array = Array("gen_votes", "--port", "0") + console.parse_args(array) must throwAn + array = Array("gen_votes", "--port", "65536") + console.parse_args(array) must throwAn[java.lang.IllegalArgumentException] + array = Array("gen_votes", "--port", "555555") + console.parse_args(array) must throwAn[java.lang.IllegalArgumentException] + } + + "read port with valid port" in + { + val console = new ConsoleImpl() + var array: Array[String] = Array("console", "--port", "40") + console.parse_args(array) must be equalTo(2) + console.port must be equalTo(40) + array = Array("gen_votes", "--port", "65535") + console.parse_args(array) must be equalTo(2) + console.port must be equalTo(65535) + } + } // parse_args + + "processPlaintextLine" should + { + "throw error on invalid line" in + { + val console = new ConsoleImpl() + console.processPlaintextLine("", 0) must throwAn[PlaintextError] + console.processPlaintextLine("d", 0) must throwAn[PlaintextError] + console.processPlaintextLine("1122", 0) must throwAn[PlaintextError] + } + + "process line with valid blank vote" in + { + val console = new ConsoleImpl() + var ballot = console.processPlaintextLine("14|", 0) + Json.toJson(ballot).toString must be equalTo("""{"id":14,"answers":[{"options":[]}]}""") + } + + "process line with valid vote" in + { + val console = new ConsoleImpl() + var ballot = console.processPlaintextLine("4888774|0,1,29||3", 0) + Json.toJson(ballot).toString must be equalTo("""{"id":4888774,"answers":[{"options":[0,1,29]},{"options":[]},{"options":[3]}]}""") + } + } // processPlaintextLine + + "get_khmac" should + { + "work" in + { + val console = new ConsoleImpl() + console.shared_secret = "" + val khmac = console.get_khmac("user_id", "obj_type", 11, "perms", Some(22)) + khmac must be equalTo(s"khmac:///sha-256;1ba07fd2e1becf9adfe74b4f6e0814fb4c9af3e0a920e718cec287857b480856/user_id:obj_type:11:perms:22") + } + } // get_khmac + + "get_election_info and get_election_info_all" should + { + val appWithRoutes = FakeApplication(withRoutes = + { + case ("GET", "/honolulu/api/election/24") => + Action + { + Ok("""{"date":"2017-04-03 09:21:04.923","payload":{"id":24,"configuration":{"id":24,"layout":"simple","director":"auth1","authorities":["auth2"],"title":"New election","description":"","questions":[{"description":"","layout":"accordion","max":1,"min":1,"num_winners":1,"title":"New question 0","tally_type":"plurality-at-large","answer_total_votes_percentage":"over-total-valid-votes","answers":[{"id":0,"category":"","details":"this is a long descriptionthis is a long descriptionthis is a long descriptionthis is a long description","sort_order":0,"urls":[],"text":"ab"},{"id":1,"category":"","details":"this is a long descriptionthis is a long descriptionthis is a long descriptionthis is a long descriptionthis is a long description","sort_order":1,"urls":[],"text":"cd"}],"extra_options":{"shuffle_categories":true,"shuffle_all_options":true,"shuffle_category_list":[],"show_points":false}}],"start_date":"2015-01-27T16:00:00.001","end_date":"2015-01-27T16:00:00.001","presentation":{"share_text":[{"network":"Twitter","button_text":"","social_message":"I have just voted in election https://go.nvotes.com, you can too! #nvotes"}],"theme":"default","urls":[],"theme_css":""},"real":false,"extra_data":"{}","virtual":false,"virtualSubelections":[],"logo_url":""},"state":"created","startDate":"2015-01-27T16:00:00.001","endDate":"2015-01-27T16:00:00.001","pks":"[{\"q\":\"24792774508736884642868649594982829646677044143456685966902090450389126928108831401260556520412635107010557472033959413182721740344201744439332485685961403243832055703485006331622597516714353334475003356107214415133930521931501335636267863542365051534250347372371067531454567272385185891163945756520887249904654258635354225185183883072436706698802915430665330310171817147030511296815138402638418197652072758525915640803066679883309656829521003317945389314422254112846989412579196000319352105328237736727287933765675623872956765501985588170384171812463052893055840132089533980513123557770728491280124996262883108653723\",\"p\":\"49585549017473769285737299189965659293354088286913371933804180900778253856217662802521113040825270214021114944067918826365443480688403488878664971371922806487664111406970012663245195033428706668950006712214428830267861043863002671272535727084730103068500694744742135062909134544770371782327891513041774499809308517270708450370367766144873413397605830861330660620343634294061022593630276805276836395304145517051831281606133359766619313659042006635890778628844508225693978825158392000638704210656475473454575867531351247745913531003971176340768343624926105786111680264179067961026247115541456982560249992525766217307447\",\"y\":\"36546166940599313416168254217744697729911998485163232812421755063790894563031509201724580104715842554805784888514324341999107803878734424284146217463747343381092445557201708138031676925435834811853723665739792579746520854169833006814210270893803218238156148054483205249064331467974912676179467375074903261138777633846359449221132575280279543204461738096325062062234143265942024786615427875783393149444592255523696808235191395788053253235712555210828055008777307102133087648043115255896201502913954041965429673426222803318582002444673454964248205507662853386137487981094735018694462667314177271930377544606339760319819\",\"g\":\"27257469383433468307851821232336029008797963446516266868278476598991619799718416119050669032044861635977216445034054414149795443466616532657735624478207460577590891079795564114912418442396707864995938563067755479563850474870766067031326511471051504594777928264027177308453446787478587442663554203039337902473879502917292403539820877956251471612701203572143972352943753791062696757791667318486190154610777475721752749567975013100844032853600120195534259802017090281900264646220781224136443700521419393245058421718455034330177739612895494553069450438317893406027741045575821283411891535713793639123109933196544017309147\"}]","real":false,"virtual":false,"logo_url":""}}""") + } + case ("GET", "/honolulu/api/election/25") => + Action + { + NotFound("ERROR") + } + }) + + "work (get_election_info)" in new WithServer(app = appWithRoutes, port = 5433) + { + val console = new ConsoleImpl() + console.http_type = "http" + console.host = "localhost" + console.port = 5433 + console.service_path = "honolulu/" + val dtoFuture24 : Future[String] = + console.get_election_info(24) map + { + dto => + Json.toJson(dto).toString + } + val dtoFuture25 : Future[String] = + console.get_election_info(25) map + { + dto => + Json.toJson(dto).toString + } + Await.ready(dtoFuture24, timeoutDuration).value.get must beSuccessfulTry + Await.ready(dtoFuture25, timeoutDuration).value.get must beFailedTry + } + + "work (get_election_info_all)" in new WithServer(app = appWithRoutes, port = 5433) + { + val console = new ConsoleImpl() + console.http_type = "http" + console.host = "localhost" + console.port = 5433 + console.service_path = "honolulu/" + val electionsSetFails = scala.collection.immutable.Set[Long](24, 25) + val electionsSetSuceeds = scala.collection.immutable.Set[Long](24) + + val dtoFutureFails = console.get_election_info_all(electionsSetFails) + val dtoFutureSuceeds = console.get_election_info_all(electionsSetSuceeds) + + Await.ready(dtoFutureFails, timeoutDuration).value.get must beFailedTry + Await.ready(dtoFutureSuceeds, timeoutDuration).value.get must beSuccessfulTry + } + } // get_election_info and get_election_info_all + + "dump_pks_elections/dump_pks" should + { + val appWithRoutes = FakeApplication(withRoutes = + { + case ("POST", "/honolulu/api/election/24/dump-pks") => + Action + { + Ok("whatever") + } + case ("POST", "/honolulu/api/election/25/dump-pks") => + Action + { + NotFound("ERROR") + } + }) + + "work (dump_pks)" in new WithServer(app = appWithRoutes, port = 5433) + { + val console = new ConsoleImpl() + console.http_type = "http" + console.host = "localhost" + console.port = 5433 + console.service_path = "honolulu/" + val dtoFuture24 = console.dump_pks(24) + val dtoFuture25 = console.dump_pks(25) + Await.ready(dtoFuture24, timeoutDuration).value.get must beSuccessfulTry + Await.ready(dtoFuture25, timeoutDuration).value.get must beFailedTry + } + + "work (dump_pks_elections)" in new WithServer(app = appWithRoutes, port = 5433) + { + val console = new ConsoleImpl() + console.http_type = "http" + console.host = "localhost" + console.port = 5433 + console.service_path = "honolulu/" + val electionsSetFails = scala.collection.immutable.Set[Long](24, 25) + val electionsSetSuceeds = scala.collection.immutable.Set[Long](24) + + val dtoFutureFails = console.dump_pks_elections(electionsSetFails) + val dtoFutureSuceeds = console.dump_pks_elections(electionsSetSuceeds) + + Await.ready(dtoFutureFails, timeoutDuration).value.get must beFailedTry + Await.ready(dtoFutureSuceeds, timeoutDuration).value.get must beSuccessfulTry + } + } // dump_pks_elections/dump_pks + + "encodePlaintext" should + { + "work" in + { + val console = new ConsoleImpl() + console.http_type = "http" + console.host = "localhost" + console.port = 5433 + console.service_path = "honolulu/" + val ballot: PlaintextBallot = + PlaintextBallot( + 42L, + Array( + PlaintextAnswer( + Array[Long](0L,1L)))) + console.encodePlaintext(ballot, dto).map{ num : BigInt => num.toLong.toString } must beEqualTo(Array("12")) + } + } // encodePlaintext + + "gen_rnd_str" should + { + "generate random strings" in + { + val console = new ConsoleImpl() + val rnd1_len2 = console.gen_rnd_str(2, "ab") + val rnd2_len2 = console.gen_rnd_str(2, "abcdefg1234567890") + val rnd_len200 = console.gen_rnd_str(200, "abcdefg1234567890") + + rnd1_len2.length must beEqualTo(2) + rnd2_len2.length must beEqualTo(2) + rnd_len200.length must beEqualTo(200) + rnd1_len2.matches("[ab]*") must beEqualTo(true) + rnd2_len2.matches("[abcdefg1234567890]*") must beEqualTo(true) + rnd_len200.matches("[abcdefg1234567890]*") must beEqualTo(true) + } + } // gen_rnd_str + + "generate_voterid" should + { + "generate random voter ids" in + { + val console = new ConsoleImpl() + console.voterid_len = 2 + val rnd_len2 = console.generate_voterid() + console.voterid_len = 200 + val rnd_len200 = console.generate_voterid() + + rnd_len2.length must beEqualTo(2) + rnd_len200.length must beEqualTo(200) + rnd_len2.matches(s"[${console.voterid_alphabet}]*") must beEqualTo(true) + rnd_len200.matches(s"[${console.voterid_alphabet}]*") must beEqualTo(true) + } + } // generate_voterid + + "generate_vote_line" should + { + val console = new ConsoleImpl() + val ballot: PlaintextBallot = + PlaintextBallot( + 42L, + Array( + PlaintextAnswer( + Array[Long](0L,1L)))) + val electionsInfoMap = scala.collection.mutable.HashMap[Long, ElectionDTO]( (42L -> dto) ) + val pksMap = console.get_pks_map(electionsInfoMap) + val pks = pksMap.get(42).get + + "work" in { + val encodedBallot = console.encodePlaintext( ballot, dto ) + val encryptedBallot = Crypto.encryptBig(pks, encodedBallot) + val line = console.generate_vote_line(42, encryptedBallot) + val match_regex = s"""42\\|[${console.voterid_alphabet}]+\\|\\{"vote":"\\{\\\\"choices\\\\":\\[\\{\\\\"alpha\\\\":\\\\"[0-9]+\\\\",\\\\"beta\\\\":\\\\"[0-9]+\\\\"\\}\\],\\\\"issue_date\\\\":\\\\"now\\\\",\\\\"proofs\\\\":\\[\\{\\\\"challenge\\\\":\\\\"[0-9]+\\\\",\\\\"commitment\\\\":\\\\"[0-9]+\\\\",\\\\"response\\\\":\\\\"[0-9]+\\\\"\\}\\]\\}","vote_hash":"[0-9a-f]+"\\}\n""" + line.matches(match_regex) must beEqualTo(true) + } + } // generate_vote_line + + "get_pks_map" should { + + "work" in { + val console = new ConsoleImpl() + val electionsInfoMap = scala.collection.mutable.HashMap[Long, ElectionDTO]( (42L -> dto) ) + val pksMap = console.get_pks_map(electionsInfoMap) + pksMap.get(45).isEmpty must beEqualTo(true) + pksMap.get(42).isEmpty must beEqualTo(false) + } + } // get_pks_map + + "generate_plaintexts_for_eid" should { + "work" in { + val console = new ConsoleImpl() + val plaintexts : String = console.generate_plaintexts_for_eid(42, 100, dto) + val regex = "[0-9]+(\\|([0-9]+,)*([0-9]+)?)*" + val linesArray = plaintexts.split("\n") + val matchesRegex = linesArray.map{ line => line.matches(regex)}.reduce(_ && _) + linesArray.size must beEqualTo(100) + matchesRegex must beEqualTo(true) + } + } + + "generate_map_num_votes_dto" should { + val dtoStr55 = +"""{ + "id": 55, + "configuration": { + "id": 55, + "layout": "simple", + "director": "auth1", + "authorities": [ + "auth2" + ], + "title": "New election", + "description": "", + "questions": [ + { + "description": "", + "layout": "accordion", + "max": 2, + "min": 1, + "num_winners": 1, + "title": "New question 0", + "tally_type": "plurality-at-large", + "answer_total_votes_percentage": "over-total-valid-votes", + "answers": [ + { + "id": 0, + "category": "", + "details": "this is a long descriptionthis is a long descriptionthis is a long descriptionthis is a long description", + "sort_order": 0, + "urls": [], + "text": "ab" + }, + { + "id": 1, + "category": "", + "details": "this is a long descriptionthis is a long descriptionthis is a long descriptionthis is a long descriptionthis is a long description", + "sort_order": 1, + "urls": [], + "text": "cd" + } + ], + "extra_options": { + "shuffle_categories": true, + "shuffle_all_options": true, + "shuffle_category_list": [], + "show_points": false + } + } + ], + "start_date": "2015-01-27T16:00:00.001", + "end_date": "2015-01-27T16:00:00.001", + "presentation": { + "share_text": [ + { + "network": "Twitter", + "button_text": "", + "social_message": "I have just voted in election https://go.nvotes.com, you can too! #nvotes" + } + ], + "theme": "default", + "urls": [], + "theme_css": "" + }, + "real": false, + "extra_data": "{}", + "virtual": false, + "virtualSubelections": [], + "logo_url": "" + }, + "state": "created", + "startDate": "2015-01-27T16:00:00.001", + "endDate": "2015-01-27T16:00:00.001", + "pks": "[{\"q\":\"24792774508736884642868649594982829646677044143456685966902090450389126928108831401260556520412635107010557472033959413182721740344201744439332485685961403243832055703485006331622597516714353334475003356107214415133930521931501335636267863542365051534250347372371067531454567272385185891163945756520887249904654258635354225185183883072436706698802915430665330310171817147030511296815138402638418197652072758525915640803066679883309656829521003317945389314422254112846989412579196000319352105328237736727287933765675623872956765501985588170384171812463052893055840132089533980513123557770728491280124996262883108653723\",\"p\":\"49585549017473769285737299189965659293354088286913371933804180900778253856217662802521113040825270214021114944067918826365443480688403488878664971371922806487664111406970012663245195033428706668950006712214428830267861043863002671272535727084730103068500694744742135062909134544770371782327891513041774499809308517270708450370367766144873413397605830861330660620343634294061022593630276805276836395304145517051831281606133359766619313659042006635890778628844508225693978825158392000638704210656475473454575867531351247745913531003971176340768343624926105786111680264179067961026247115541456982560249992525766217307447\",\"y\":\"36546166940599313416168254217744697729911998485163232812421755063790894563031509201724580104715842554805784888514324341999107803878734424284146217463747343381092445557201708138031676925435834811853723665739792579746520854169833006814210270893803218238156148054483205249064331467974912676179467375074903261138777633846359449221132575280279543204461738096325062062234143265942024786615427875783393149444592255523696808235191395788053253235712555210828055008777307102133087648043115255896201502913954041965429673426222803318582002444673454964248205507662853386137487981094735018694462667314177271930377544606339760319819\",\"g\":\"27257469383433468307851821232336029008797963446516266868278476598991619799718416119050669032044861635977216445034054414149795443466616532657735624478207460577590891079795564114912418442396707864995938563067755479563850474870766067031326511471051504594777928264027177308453446787478587442663554203039337902473879502917292403539820877956251471612701203572143972352943753791062696757791667318486190154610777475721752749567975013100844032853600120195534259802017090281900264646220781224136443700521419393245058421718455034330177739612895494553069450438317893406027741045575821283411891535713793639123109933196544017309147\"}]", + "real": false, + "virtual": false, + "logo_url": "" +}""" + val dto55 : ElectionDTO = Json.parse(dtoStr55).validate[ElectionDTO].get + "work" in { + val console = new ConsoleImpl() + val electionsInfoMap = + scala.collection.mutable.HashMap[Long, ElectionDTO]( + (42L -> dto), + (55L -> dto55)) + console.vote_count = 5 + val numVotesMap = console.generate_map_num_votes_dto(electionsInfoMap) + numVotesMap.get(42).get._1 must beEqualTo(3) + numVotesMap.get(55).get._1 must beEqualTo(2) + } + } // generate_map_num_votes_dto + +} // ConsoleSpec diff --git a/test/ElectionsSpec.scala b/test/ElectionsSpec.scala new file mode 100644 index 00000000..43dac66b --- /dev/null +++ b/test/ElectionsSpec.scala @@ -0,0 +1,116 @@ +/** + * This file is part of agora_elections. + * Copyright (C) 2014-2016 Agora Voting SL + + * agora_elections is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License. + + * agora_elections is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + + * You should have received a copy of the GNU Affero General Public License + * along with agora_elections. If not, see . +**/ +package test + +import controllers.routes +import utils.Response +import utils.JsonFormatters._ + +import org.specs2.mutable._ +import org.specs2.runner._ +import org.junit.runner._ + +import play.api.test._ +import play.api.test.Helpers._ +import play.api.libs.json.Json +import play.api.libs.json.Reads +import play.api.libs.json.JsSuccess + +import scala.util.Try +import scala.util.Failure +import scala.util.Success + +import scala.concurrent._ + +import models._ +import play.api.db.slick.DB + + +@RunWith(classOf[JUnitRunner]) +class ElectionsSpec extends Specification with TestContexts with Response { + + "ElectionsApi" should { + + "reject bad auth" in new AppWithDbData() { + + val response = route(FakeRequest(POST, routes.ElectionsApi.register(1).url) + .withJsonBody(TestData.config) + .withHeaders(("Authorization", "bogus")) + ).get + + status(response) must equalTo(FORBIDDEN) + } + + "allow registering an retrieving an election" in new AppWithDbData() { + + // for this to work we need to set the pk for the election manually (for election 1020) + var response = route(FakeRequest(POST, routes.ElectionsApi.register(1).url) + .withJsonBody(TestData.config) + .withHeaders(("Authorization", getAuth("", "AuthEvent", 1, "edit"))) + ).get + + responseCheck(response, (r:Response[Int]) => r.payload > 0) + + response = route(FakeRequest(GET, routes.ElectionsApi.get(1).url) + .withJsonBody(TestData.config) + // .withHeaders(("Authorization", getAuth("", "election", 0, "admin"))) + ).get + + responseCheck(response, (r:Response[ElectionDTO]) => r.payload.configuration.title == "Votación de candidatos") + } + + "allow updating an election" in new AppWithDbData() { + + DB.withSession { implicit session => + val cfg = TestData.config.validate[ElectionConfig].get + Elections.insert(Election(cfg.id, TestData.config.toString, Elections.REGISTERED, cfg.start_date, cfg.end_date, None, None, None, None)) + } + + val response = route(FakeRequest(POST, routes.ElectionsApi.update(1).url) + .withJsonBody(TestData.config) + .withHeaders(("Authorization", getAuth("", "AuthEvent", 1, "edit"))) + ).get + + responseCheck(response, (r:Response[Int]) => r.payload > 0) + } + + "allow retrieving authority data" in new AppWithDbData() { + + val response = route(FakeRequest(GET, routes.ElectionsApi.getAuthorities.url) + .withHeaders(("Authorization", "bogus")) + ).get + + responseCheck(response, (r:Response[Map[String, AuthData]]) => r.payload.size == 2) + } + } + + def responseCheck[T: Reads](result: Future[play.api.mvc.Result], f: T => Boolean, code:Int = OK) = { + println(s">>> received '${contentAsString(result)}'") + + status(result) must equalTo(code) + contentType(result) must beSome.which(_ == "application/json") + + val parsed = Try(Json.parse(contentAsString(result))).map(_.validate[T]) + // force it to crash if parse errors + parsed.get + + parsed must beLike { + case Success(JsSuccess(response, _)) if f(response) => ok + case _ => ko + } + } +} \ No newline at end of file diff --git a/test/MiscSpec.scala b/test/MiscSpec.scala new file mode 100644 index 00000000..6c18aa29 --- /dev/null +++ b/test/MiscSpec.scala @@ -0,0 +1,53 @@ +/** + * This file is part of agora_elections. + * Copyright (C) 2014-2016 Agora Voting SL + + * agora_elections is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License. + + * agora_elections is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + + * You should have received a copy of the GNU Affero General Public License + * along with agora_elections. If not, see . +**/ +package test + +import org.specs2.mutable._ +import org.specs2.runner._ +import org.junit.runner._ + +import play.api.test._ +import play.api.test.Helpers._ +import play.api.libs.json.Json +import play.api.libs.json.Reads +import play.api.libs.json.JsSuccess + +import scala.concurrent._ +import scala.util.Try +import scala.util.Failure +import scala.util.Success + +import controllers.routes +import utils._ + +// FIXME add legendre tests +/** */ +@RunWith(classOf[JUnitRunner]) +class MiscSpec extends Specification { + + "Misc utils" should { + + "allow correct hmac calculation" in { + Crypto.hmac("pajarraco", "forrarme:1417267291") must beEqualTo("d5c4e961a496559b0a19039bc9cb62a3d9b63b38c54eb14286ca54364d21841e") + } + + /** this does not work as the dumped votes vary each test, you can test sha256 using echo "blabla" | sha512sum to verify */ + /* "allow correct hashing of file" in new WithApplication() { + Datastore.hashVotes(1) must beEqualTo("cf53a19dc191e3af395e463b8664578d4daa8770c4e98d0ab863ffdd47784f18") + } */ + } +} \ No newline at end of file From db7cca52cb95ea93f1d43ff833ff4499b12bbfe6 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 4 Apr 2017 10:51:01 +0200 Subject: [PATCH 60/68] add another test :) --- test/ConsoleSpec.scala | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/ConsoleSpec.scala b/test/ConsoleSpec.scala index f0170668..6b005f25 100644 --- a/test/ConsoleSpec.scala +++ b/test/ConsoleSpec.scala @@ -35,6 +35,11 @@ import scala.util._ import play.api.cache.Cache import java.sql.Timestamp +import play.api.libs.Files._ +import java.nio.file.{Paths, Files, Path} +import java.nio.file.StandardOpenOption._ +import java.nio.charset.StandardCharsets + @RunWith(classOf[JUnitRunner]) class ConsoleSpec extends Specification with TestContexts @@ -496,4 +501,23 @@ class ConsoleSpec extends Specification with TestContexts } } // generate_map_num_votes_dto + "getElectionsSet" should { + "work" in { + val tempFile = TemporaryFile(prefix = "election_ids", suffix = ".txt") + val filePath = tempFile.file.getAbsolutePath() + val eidsContent = "42\n55" + Files.write(Paths.get(filePath), eidsContent.getBytes(StandardCharsets.UTF_8), APPEND) + val console = new ConsoleImpl() + console.election_ids_path = filePath + val futureElectionSet = console.getElectionsSet() + val optionElectionSet = Await.ready(futureElectionSet, timeoutDuration).value + + optionElectionSet.get must beSuccessfulTry + val electionSet = optionElectionSet.get.get + electionSet.contains(42) must beEqualTo(true) + electionSet.contains(55) must beEqualTo(true) + electionSet.size must beEqualTo(2) + } + } + } // ConsoleSpec From dee59aba2f0485d62e39ef50b1671de5e2ea72e6 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 4 Apr 2017 11:54:04 +0200 Subject: [PATCH 61/68] remove gen_votes and send_votes from admin.py as Console replaces them --- admin/admin.py | 168 +---------------------------------------- admin/plaintexts.txt | 2 - test/ConsoleSpec.scala | 2 +- 3 files changed, 2 insertions(+), 170 deletions(-) delete mode 100644 admin/plaintexts.txt diff --git a/admin/admin.py b/admin/admin.py index aca7e5e1..307c3901 100644 --- a/admin/admin.py +++ b/admin/admin.py @@ -731,170 +731,6 @@ def change_social(cfg, args): print("invalid share-config file %s" % args.share_config) return 400 -def gen_votes(cfg, args): - def _open(path, mode): - return codecs.open(path, encoding='utf-8', mode=mode) - - def _read_file(path): - _check_file(path) - with _open(path, mode='r') as f: - return f.read() - - def _check_file(path): - if not os.access(path, os.R_OK): - raise Exception("Error: can't read %s" % path) - if not os.path.isfile(path): - raise Exception("Error: not a file %s" % path) - - def _write_file(path, data): - with _open(path, mode='w') as f: - return f.write(data) - - def gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count): - base_plaintexts = [ d.strip() for d in _read_file(base_plaintexts_path).splitlines() ] - new_plaintext_path = os.path.join(temp_path, 'plaintext') - new_plaintext = "[\n" - counter = 0 - mod_base = len(base_plaintexts) - while counter < vote_count: - base_ballot = base_plaintexts[counter % mod_base] - json_ballot = "[%s]" % base_ballot - if counter + 1 == vote_count: - new_plaintext += "%s]" % json_ballot - else: - new_plaintext += "%s,\n" % json_ballot - counter += 1 - _write_file(new_plaintext_path, new_plaintext) - return new_plaintext_path - - if args.vote_count <= 0: - raise Exception("vote count must be > 0") - - with TemporaryDirectory() as temp_path: - print("%s created temporary folder at %s" % (str(datetime.now()), temp_path)) - - election_id = cfg['election_id'] - vote_count = args.vote_count - cfg['encrypt-count'] = vote_count - - if cfg['ciphertexts'] is None: - raise Exception("missing ciphertexts argument") - - if cfg['plaintexts'] is None: - raise Exception("missing ciphertexts argument") - - _check_file(cfg['plaintexts']) - save_ciphertexts_path = cfg['ciphertexts'] - - # a list of base plaintexts to generate ballots - base_plaintexts_path = cfg["plaintexts"] - ciphertexts_path = os.path.join(temp_path, 'ciphertexts') - cfg['ciphertexts'] = ciphertexts_path - print("%s start generation of plaintexts" % str(datetime.now())) - cfg["plaintexts"] = gen_all_plaintexts(temp_path, base_plaintexts_path, vote_count) - print("%s plaintexts created" % str(datetime.now())) - print("%s start dump_pks" % str(datetime.now())) - dump_pks(cfg, args) - print("%s pks dumped" % str(datetime.now())) - print("%s start ballot encryption" % str(datetime.now())) - start_time = time.time() - encrypt(cfg, args) - print("%s ballots encrypted" % str(datetime.now())) - end_time = time.time() - delta_t = end_time - start_time - troughput = vote_count / float(delta_t) - print("encrypted %i votes in %f secs. %f votes/sec" % (vote_count, delta_t, troughput)) - - shutil.copy2(ciphertexts_path, save_ciphertexts_path) - print("encrypted ballots saved to file %s" % ciphertexts_path) - -def send_votes(cfg, args): - def _open(path, mode): - return codecs.open(path, encoding='utf-8', mode=mode) - - def _check_file(path): - if not os.access(path, os.R_OK): - raise Exception("Error: can't read %s" % path) - if not os.path.isfile(path): - raise Exception("Error: not a file %s" % path) - - def _read_file(path): - _check_file(path) - with _open(path, mode='r') as f: - return f.read() - - def gen_rnd_str(length, choices): - return ''.join( - random.SystemRandom().choice(choices) - for _ in range(length) - ) - - def gen_all_khmacs(vote_count, election_id): - import hmac - start_time = time.time() - alphabet = '0123456789abcdef' - counter = 0 - khmac_list = [] - voterid_len = 28 - while counter < vote_count: - voterid = gen_rnd_str(voterid_len, alphabet) - khmac = get_hmac(cfg, voterid, "AuthEvent", election_id, 'vote') - khmac_list.append((voterid, khmac)) - counter += 1 - - end_time = time.time() - delta_t = end_time - start_time - troughput = vote_count / float(delta_t) - print("created %i khmacs in %f secs. %f khmacs/sec" % (vote_count, delta_t, troughput)) - - return khmac_list - - def send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id): - start_time = time.time() - cyphertexts_json = json.loads(_read_file(ciphertexts_path)) - host,port = get_local_hostport() - for index, vote in enumerate(cyphertexts_json): - vote_string = json.dumps(vote) - if index >= vote_count: - break - voterid, khmac = khmac_list[index] - headers = { - 'content-type': 'application/json', - 'Authorization': khmac - } - vote_hash = hashlib.sha256(str.encode(vote_string)).hexdigest() - ballot = json.dumps({ - "vote": vote_string, - "vote_hash": vote_hash - }) - url = 'http://%s:%d/api/election/%i/voter/%s' % (host, port, election_id, voterid) - r = requests.post(url, data=ballot, headers=headers) - if r.status_code != 200: - raise Exception("Error voting: HTTP POST to %s with khmac %s returned code %i, error %s, http data: %s" % (url, khmac, r.status_code, r.text[:200], ballot)) - end_time = time.time() - delta_t = end_time - start_time - troughput = vote_count / float(delta_t) - print("sent %i votes in %f secs. %f votes/sec" % (vote_count, delta_t, troughput)) - - if args.vote_count <= 0: - raise Exception("vote count must be > 0") - - if cfg['ciphertexts'] is None: - raise Exception("ciphertexts argument is missing") - - _check_file(cfg['ciphertexts']) - ciphertexts_path = cfg['ciphertexts'] - - election_id = cfg['election_id'] - vote_count = args.vote_count - - print("%s start khmac generation" % str(datetime.now())) - khmac_list = gen_all_khmacs(vote_count, election_id) - print("%s khmacs generated" % str(datetime.now())) - print("%s start sending ballots" % str(datetime.now())) - send_all_ballots(vote_count, ciphertexts_path, khmac_list, election_id) - print("%s ballots sent" % str(datetime.now())) - def get_hmac(cfg, userId, objType, objId, perm): import hmac @@ -936,9 +772,7 @@ def main(argv): encryptNode : encrypts votes using node (public key must be in datastore) dump_votes : dumps votes for an election (private datastore) change_social : changes the social netoworks share buttons configuration -authapi_ensure_acls --acls-path : ensure that the acls inside acl_path exist. -gen_votes : generate votes -send_votes : send votes generated by the command gen_votes +authapi_ensure_acls --acls-path : ensure that the acls inside acl_path exist ''') parser.add_argument('--ciphertexts', help='file to write ciphertetxs (used in dump, load and encrypt)') parser.add_argument('--acls-path', help='''the file has one line per acl with format: '(email:email@example.com|tlf:+34666777888),permission_name,object_type,object_id,user_election_id' ''') diff --git a/admin/plaintexts.txt b/admin/plaintexts.txt deleted file mode 100644 index d5bc45cb..00000000 --- a/admin/plaintexts.txt +++ /dev/null @@ -1,2 +0,0 @@ -22|56,2,39||5 -39|0|1 \ No newline at end of file diff --git a/test/ConsoleSpec.scala b/test/ConsoleSpec.scala index 6b005f25..667376aa 100644 --- a/test/ConsoleSpec.scala +++ b/test/ConsoleSpec.scala @@ -517,7 +517,7 @@ class ConsoleSpec extends Specification with TestContexts electionSet.contains(42) must beEqualTo(true) electionSet.contains(55) must beEqualTo(true) electionSet.size must beEqualTo(2) - } + } // getElectionsSet } } // ConsoleSpec From 03423c8029e2564e978595158d941cfbbe31a716 Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 4 Apr 2017 13:27:23 +0200 Subject: [PATCH 62/68] remove odd file --- app/what.csv | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 app/what.csv diff --git a/app/what.csv b/app/what.csv deleted file mode 100644 index ae14f4b8..00000000 --- a/app/what.csv +++ /dev/null @@ -1,2 +0,0 @@ -hello world file -dd \ No newline at end of file From 9af85d5ee6e8e65f55b95534322f03721811ddfa Mon Sep 17 00:00:00 2001 From: Findeton Date: Tue, 4 Apr 2017 15:01:31 +0200 Subject: [PATCH 63/68] delete console.sh --- admin/console.sh | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100755 admin/console.sh diff --git a/admin/console.sh b/admin/console.sh deleted file mode 100755 index ffd88f50..00000000 --- a/admin/console.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh - -# This file is part of agora_elections. -# Copyright (C) 2014-2016 Agora Voting SL - -# agora_elections is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License. - -# agora_elections is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with agora_elections. If not, see . - -IVY=~/.ivy2/cache -AGORA_HOME=.. -java -classpath $IVY/cache/com.typesafe.akka/akka-actor_2.11/jars/akka-actor_2.11-2.3.4.jar:$AGORA_HOME/target/scala-2.11/agora-elections_2.11-1.0-SNAPSHOT.jar:$IVY/com.typesafe.play/play-json_2.11/jars/play-json_2.11-2.3.6.jar:$IVY/org.scala-lang/scala-library/jars/scala-library-2.11.1.jar:$IVY/com.fasterxml.jackson.core/jackson-core/bundles/jackson-core-2.3.3.jar:$IVY/com.fasterxml.jackson.core/jackson-databind/bundles/jackson-databind-2.3.3.jar:$IVY/com.fasterxml.jackson.core/jackson-annotations/bundles/jackson-annotations-2.3.2.jar:$IVY/com.typesafe.play/play-functional_2.11/jars/play-functional_2.11-2.3.6.jar:$IVY/org.joda/joda-convert/jars/joda-convert-1.6.jar:$IVY/joda-time/joda-time/jars/joda-time-2.3.jar commands.Console $* - From 5f2bc95d9a906ef61e456d7ec82df90175209513 Mon Sep 17 00:00:00 2001 From: Findeton Date: Wed, 5 Apr 2017 11:48:12 +0200 Subject: [PATCH 64/68] add config params for download tally retries and timeout --- app/controllers/ElectionsApi.scala | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/app/controllers/ElectionsApi.scala b/app/controllers/ElectionsApi.scala index 19ba2a52..8cb44087 100644 --- a/app/controllers/ElectionsApi.scala +++ b/app/controllers/ElectionsApi.scala @@ -79,6 +79,8 @@ object ElectionsApi val slickExecutionContext = Akka.system.dispatchers.lookup("play.akka.actor.slick-context") val allowPartialTallies = Play.current.configuration.getBoolean("app.partial-tallies").getOrElse(false) val authorities = getAuthorityData + val download_tally_timeout = Play.current.configuration.getInt("app.download_tally_timeout").get + val download_tally_retries = Play.current.configuration.getInt("app.download_tally_retries").get /** inserts election into the db in the registered state */ def register(id: Long) = HAction("", "AuthEvent", id, "edit").async(BodyParsers.parse.json) { request => @@ -716,10 +718,32 @@ object ElectionsApi Logger.info(s"downloading tally from $url") + def retryWrapper( + wsRequest: WSRequestHolder, + f: Future[(WSResponseHeaders, Enumerator[Array[Byte]])], + times: Int) + : + Future[(WSResponseHeaders, Enumerator[Array[Byte]])] = + { + f.recoverWith { + case t : Throwable => + val promise = Promise[(WSResponseHeaders, Enumerator[Array[Byte]])]() + if (times > 0) + { + promise completeWith retryWrapper(wsRequest, wsRequest.getStream(), times - 1) + } + else + { + promise failure t + } + promise.future + }(slickExecutionContext) + } + // taken from https://www.playframework.com/documentation/2.3.x/ScalaWS + val wsRequest = WS.url(url).withRequestTimeout(download_tally_timeout) val futureResponse: Future[(WSResponseHeaders, Enumerator[Array[Byte]])] = - /* 3600 seconds timeout should be enough for everybody WCPGR? */ - WS.url(url).withRequestTimeout(3600000).getStream() + retryWrapper(wsRequest, wsRequest.getStream(), download_tally_retries) val downloadedFile: Future[Unit] = futureResponse.flatMap { case (headers, body) => From f9ed2d338cce4a3e387f6ea73d2542c7863c1ea2 Mon Sep 17 00:00:00 2001 From: Findeton Date: Wed, 5 Apr 2017 13:08:42 +0200 Subject: [PATCH 65/68] add some comments --- app/controllers/ElectionsApi.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/controllers/ElectionsApi.scala b/app/controllers/ElectionsApi.scala index 8cb44087..ea917b2e 100644 --- a/app/controllers/ElectionsApi.scala +++ b/app/controllers/ElectionsApi.scala @@ -718,6 +718,7 @@ object ElectionsApi Logger.info(s"downloading tally from $url") + // function to retry the http request a number of times def retryWrapper( wsRequest: WSRequestHolder, f: Future[(WSResponseHeaders, Enumerator[Array[Byte]])], @@ -741,7 +742,9 @@ object ElectionsApi } // taken from https://www.playframework.com/documentation/2.3.x/ScalaWS + // configure http request val wsRequest = WS.url(url).withRequestTimeout(download_tally_timeout) + // http request future (including retries) val futureResponse: Future[(WSResponseHeaders, Enumerator[Array[Byte]])] = retryWrapper(wsRequest, wsRequest.getStream(), download_tally_retries) From 93d49ea94fdb0c27959ed2d5949e99a13c65c136 Mon Sep 17 00:00:00 2001 From: Findeton Date: Wed, 5 Apr 2017 16:44:48 +0200 Subject: [PATCH 66/68] fix for case class with >= 22 fields for play 2.3.x --- app/models/Models.scala | 4 +- app/utils/JsonFormatters.scala | 107 ++++++++++++++++++++++++++++++++- 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/app/models/Models.scala b/app/models/Models.scala index dab2ee47..5e579927 100644 --- a/app/models/Models.scala +++ b/app/models/Models.scala @@ -425,7 +425,7 @@ case class QuestionExtra( shuffle_category_list: Option[Array[String]], show_points: Option[Boolean], default_selected_option_ids: Option[Array[Int]], - select_categories_1click: Option[Boolean]), + select_categories_1click: Option[Boolean], answer_columns_size: Option[String], group_answer_pairs: Option[String]) { @@ -454,7 +454,7 @@ case class QuestionExtra( assert(!answer_columns_size.isDefined || List(12,6,4,3).contains(answer_columns_size.get.toInt), "invalid answer_columns_size, can only be a string with 12,6,4,3") assert(!group_answer_pairs.isDefined || group_answer_pairs.get.length <= SHORT_STRING, "group_answer_pairs too long") - + assert(!(shuffle_all_options.isDefined && shuffle_category_list.isDefined && shuffle_all_options.get) || 0 == shuffle_category_list.get.size, "shuffle_all_options is true but shuffle_category_list is not empty") diff --git a/app/utils/JsonFormatters.scala b/app/utils/JsonFormatters.scala index d82367dc..4d6786dd 100644 --- a/app/utils/JsonFormatters.scala +++ b/app/utils/JsonFormatters.scala @@ -63,7 +63,112 @@ object JsonFormatters { implicit val urlF = Json.format[Url] implicit val answerF = Json.format[Answer] - implicit val questionExtraF = Json.format[QuestionExtra] + + ////////////////////////////////////////////////////////////////////////////// + // this is not pretty but at least it works + // it's necessary for case classes with >= 22 fields + // for this version of Play (2.3.6) + // implicit val questionExtraF = Json.format[QuestionExtra] + // See http://stackoverflow.com/questions/28167971/scala-case-having-22-fields-but-having-issue-with-play-json-in-scala-2-11-5 + val questionExtraFirstF + : + OFormat[( + Option[String], Option[String], Option[String], Option[String], + Option[String], Option[String], Option[String], Option[String], + Option[String], Option[String], Option[String], Option[String], + Option[String], Option[String], Option[String]) + ] = + ( + (__ \ "group").format[Option[String]] and + (__ \ "next_button").format[Option[String]] and + (__ \ "shuffled_categories").format[Option[String]] and + (__ \ "shuffling_policy").format[Option[String]] and + (__ \ "ballot_parity_criteria").format[Option[String]] and + (__ \ "restrict_choices_by_tag__name").format[Option[String]] and + (__ \ "restrict_choices_by_tag__max").format[Option[String]] and + (__ \ "restrict_choices_by_tag__max_error_msg").format[Option[String]] and + (__ \ "accordion_folding_policy").format[Option[String]] and + (__ \ "restrict_choices_by_no_tag__max").format[Option[String]] and + (__ \ "force_allow_blank_vote").format[Option[String]] and + (__ \ "recommended_preset__tag").format[Option[String]] and + (__ \ "recommended_preset__title").format[Option[String]] and + (__ \ "recommended_preset__accept_text").format[Option[String]] and + (__ \ "recommended_preset__deny_text").format[Option[String]] + ).tupled + + val questionExtraSecondF + : + OFormat[( + Option[Boolean], Option[Boolean], Option[Array[String]], + Option[Boolean], Option[Array[Int]], Option[Boolean], Option[String], + Option[String]) + ] = + ( + (__ \ "shuffle_categories").format[Option[Boolean]] and + (__ \ "shuffle_all_options").format[Option[Boolean]] and + (__ \ "shuffle_category_list").format[Option[Array[String]]] and + (__ \ "show_points").format[Option[Boolean]] and + (__ \ "default_selected_option_ids").format[Option[Array[Int]]] and + (__ \ "select_categories_1click").format[Option[Boolean]] and + (__ \ "answer_columns_size").format[Option[String]] and + (__ \ "group_answer_pairs").format[Option[String]] + ).tupled + + implicit val questionExtraF : Format[QuestionExtra] = + ( questionExtraFirstF and questionExtraSecondF ).apply( + { + case ( + ( + group, next_button, shuffled_categories, shuffling_policy, + ballot_parity_criteria, restrict_choices_by_tag__name, + restrict_choices_by_tag__max, restrict_choices_by_tag__max_error_msg, + accordion_folding_policy, restrict_choices_by_no_tag__max, + force_allow_blank_vote, recommended_preset__tag, + recommended_preset__title, recommended_preset__accept_text, + recommended_preset__deny_text + ), + ( + shuffle_categories, shuffle_all_options, shuffle_category_list, + show_points, default_selected_option_ids, select_categories_1click, + answer_columns_size, group_answer_pairs + ) + ) => + QuestionExtra( + group, next_button, shuffled_categories, shuffling_policy, + ballot_parity_criteria, restrict_choices_by_tag__name, + restrict_choices_by_tag__max, restrict_choices_by_tag__max_error_msg, + accordion_folding_policy, restrict_choices_by_no_tag__max, + force_allow_blank_vote, recommended_preset__tag, + recommended_preset__title, recommended_preset__accept_text, + recommended_preset__deny_text, + shuffle_categories, shuffle_all_options, shuffle_category_list, + show_points, default_selected_option_ids, select_categories_1click, + answer_columns_size, group_answer_pairs + ) + }, + q + => + ( + ( + q.group, q.next_button, q.shuffled_categories, q.shuffling_policy, + q.ballot_parity_criteria, q.restrict_choices_by_tag__name, + q.restrict_choices_by_tag__max, q.restrict_choices_by_tag__max_error_msg, + q.accordion_folding_policy, q.restrict_choices_by_no_tag__max, + q.force_allow_blank_vote, q.recommended_preset__tag, + q.recommended_preset__title, q.recommended_preset__accept_text, + q.recommended_preset__deny_text + ), + ( + q.shuffle_categories, q.shuffle_all_options, q.shuffle_category_list, + q.show_points, q.default_selected_option_ids, q.select_categories_1click, + q.answer_columns_size, q.group_answer_pairs + ) + ) + ) + + ////////////////////////////////////////////////////////////////////////////// + + implicit val questionF = Json.format[Question] implicit val ShareTextItemF = Json.format[ShareTextItem] From 19dddee2ac98b5b90b03ab37a05ab78a23ffc513 Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 6 Apr 2017 12:37:01 +0200 Subject: [PATCH 67/68] using shapeless for json parsing >21 field case classes: previous hack compiled but was incompatible with Option and parsing to None values --- app/utils/JsonFormatters.scala | 166 ++++++++++++++------------------- build.sbt | 3 +- 2 files changed, 71 insertions(+), 98 deletions(-) diff --git a/app/utils/JsonFormatters.scala b/app/utils/JsonFormatters.scala index 4d6786dd..d9d04e79 100644 --- a/app/utils/JsonFormatters.scala +++ b/app/utils/JsonFormatters.scala @@ -22,6 +22,13 @@ import models._ import java.sql.Timestamp import scala.math.BigInt + +import play.api.libs._ +import json._ + +import shapeless.{ `::` => :#:, _ } +import poly._ + /** * Formatters for json parsing and writing * @@ -66,106 +73,71 @@ object JsonFormatters { ////////////////////////////////////////////////////////////////////////////// // this is not pretty but at least it works - // it's necessary for case classes with >= 22 fields + // it's necessary for case classes with >= 22 fields // for this version of Play (2.3.6) - // implicit val questionExtraF = Json.format[QuestionExtra] - // See http://stackoverflow.com/questions/28167971/scala-case-having-22-fields-but-having-issue-with-play-json-in-scala-2-11-5 - val questionExtraFirstF - : - OFormat[( - Option[String], Option[String], Option[String], Option[String], - Option[String], Option[String], Option[String], Option[String], - Option[String], Option[String], Option[String], Option[String], - Option[String], Option[String], Option[String]) - ] = - ( - (__ \ "group").format[Option[String]] and - (__ \ "next_button").format[Option[String]] and - (__ \ "shuffled_categories").format[Option[String]] and - (__ \ "shuffling_policy").format[Option[String]] and - (__ \ "ballot_parity_criteria").format[Option[String]] and - (__ \ "restrict_choices_by_tag__name").format[Option[String]] and - (__ \ "restrict_choices_by_tag__max").format[Option[String]] and - (__ \ "restrict_choices_by_tag__max_error_msg").format[Option[String]] and - (__ \ "accordion_folding_policy").format[Option[String]] and - (__ \ "restrict_choices_by_no_tag__max").format[Option[String]] and - (__ \ "force_allow_blank_vote").format[Option[String]] and - (__ \ "recommended_preset__tag").format[Option[String]] and - (__ \ "recommended_preset__title").format[Option[String]] and - (__ \ "recommended_preset__accept_text").format[Option[String]] and - (__ \ "recommended_preset__deny_text").format[Option[String]] - ).tupled - - val questionExtraSecondF - : - OFormat[( - Option[Boolean], Option[Boolean], Option[Array[String]], - Option[Boolean], Option[Array[Int]], Option[Boolean], Option[String], - Option[String]) - ] = - ( - (__ \ "shuffle_categories").format[Option[Boolean]] and - (__ \ "shuffle_all_options").format[Option[Boolean]] and - (__ \ "shuffle_category_list").format[Option[Array[String]]] and - (__ \ "show_points").format[Option[Boolean]] and - (__ \ "default_selected_option_ids").format[Option[Array[Int]]] and - (__ \ "select_categories_1click").format[Option[Boolean]] and - (__ \ "answer_columns_size").format[Option[String]] and - (__ \ "group_answer_pairs").format[Option[String]] - ).tupled - - implicit val questionExtraF : Format[QuestionExtra] = - ( questionExtraFirstF and questionExtraSecondF ).apply( - { - case ( - ( - group, next_button, shuffled_categories, shuffling_policy, - ballot_parity_criteria, restrict_choices_by_tag__name, - restrict_choices_by_tag__max, restrict_choices_by_tag__max_error_msg, - accordion_folding_policy, restrict_choices_by_no_tag__max, - force_allow_blank_vote, recommended_preset__tag, - recommended_preset__title, recommended_preset__accept_text, - recommended_preset__deny_text - ), - ( - shuffle_categories, shuffle_all_options, shuffle_category_list, - show_points, default_selected_option_ids, select_categories_1click, - answer_columns_size, group_answer_pairs - ) - ) => - QuestionExtra( - group, next_button, shuffled_categories, shuffling_policy, - ballot_parity_criteria, restrict_choices_by_tag__name, - restrict_choices_by_tag__max, restrict_choices_by_tag__max_error_msg, - accordion_folding_policy, restrict_choices_by_no_tag__max, - force_allow_blank_vote, recommended_preset__tag, - recommended_preset__title, recommended_preset__accept_text, - recommended_preset__deny_text, - shuffle_categories, shuffle_all_options, shuffle_category_list, - show_points, default_selected_option_ids, select_categories_1click, - answer_columns_size, group_answer_pairs - ) - }, - q - => - ( - ( - q.group, q.next_button, q.shuffled_categories, q.shuffling_policy, - q.ballot_parity_criteria, q.restrict_choices_by_tag__name, - q.restrict_choices_by_tag__max, q.restrict_choices_by_tag__max_error_msg, - q.accordion_folding_policy, q.restrict_choices_by_no_tag__max, - q.force_allow_blank_vote, q.recommended_preset__tag, - q.recommended_preset__title, q.recommended_preset__accept_text, - q.recommended_preset__deny_text - ), - ( - q.shuffle_categories, q.shuffle_all_options, q.shuffle_category_list, - q.show_points, q.default_selected_option_ids, q.select_categories_1click, - q.answer_columns_size, q.group_answer_pairs - ) + // it requires shapeless 2.0.0 (newer shapelessversions may not be compatible + // with this code) + // from https://gist.github.com/negator/5139ddb5f6d91cbe7b0c + // http://stackoverflow.com/questions/23571677/22-fields-limit-in-scala-2-11-play-framework-2-3-case-classes-and-functions + + + implicit val writesInstance: LabelledProductTypeClass[Writes] = new LabelledProductTypeClass[Writes] { + + def emptyProduct: Writes[HNil] = Writes(_ => Json.obj()) + + def product[F, T <: HList](name: String, FHead: Writes[F], FTail: Writes[T]) = Writes[F :#: T] { + case head :#: tail => + val h = FHead.writes(head) + val t = FTail.writes(tail) + + (h, t) match { + case (JsNull, t: JsObject) => t + case (h: JsValue, t: JsObject) => Json.obj(name -> h) ++ t + case _ => Json.obj() + } + } + def project[F, G](instance: => Writes[G], to: F => G, from: G => F) = Writes[F](f => instance.writes(to(f))) + } + object SWrites extends LabelledProductTypeClassCompanion[Writes] + + implicit val readsInstance: LabelledProductTypeClass[Reads] = new LabelledProductTypeClass[Reads] { + + def emptyProduct: Reads[HNil] = Reads(_ => JsSuccess(HNil)) + + def product[F, T <: HList](name: String, FHead: Reads[F], FTail: Reads[T]) = Reads[F :#: T] { + case obj @ JsObject(fields) => + for { + head <- FHead.reads(obj \ name) + tail <- FTail.reads(obj - name) + } yield head :: tail + + case _ => JsError("Json object required") + } + + def project[F, G](instance: => Reads[G], to: F => G, from: G => F) = Reads[F](instance.map(from).reads) + } + object SReads extends LabelledProductTypeClassCompanion[Reads] + + implicit val formatInstance: LabelledProductTypeClass[Format] = new LabelledProductTypeClass[Format] { + def emptyProduct: Format[HNil] = Format( + readsInstance.emptyProduct, + writesInstance.emptyProduct + ) + + def product[F, T <: HList](name: String, FHead: Format[F], FTail: Format[T]) = Format[F :#: T] ( + readsInstance.product[F, T](name, FHead, FTail), + writesInstance.product[F, T](name, FHead, FTail) ) - ) + def project[F, G](instance: => Format[G], to: F => G, from: G => F) = Format[F]( + readsInstance.project(instance, to, from), + writesInstance.project(instance, to, from) + ) + } + object SFormats extends LabelledProductTypeClassCompanion[Format] + + implicit val questionExtraWrites : Writes[QuestionExtra] = SWrites.auto.derive[QuestionExtra] + implicit val questionExtraReads : Reads[QuestionExtra] = SReads.auto.derive[QuestionExtra] ////////////////////////////////////////////////////////////////////////////// diff --git a/build.sbt b/build.sbt index f234d4eb..cae36b66 100644 --- a/build.sbt +++ b/build.sbt @@ -38,7 +38,8 @@ libraryDependencies ++= Seq( "org.bouncycastle" % "bcprov-jdk16" % "1.45", "com.googlecode.owasp-java-html-sanitizer" % "owasp-java-html-sanitizer" % "r239", "commons-validator" % "commons-validator" % "1.4.1", - "com.github.mumoshu" %% "play2-memcached-play23" % "0.7.0" + "com.github.mumoshu" %% "play2-memcached-play23" % "0.7.0", + "com.chuusai" %% "shapeless" % "2.0.0" ) resolvers += "Spy Repository" at "http://files.couchbase.com/maven2" // required to resolve `spymemcached`, the plugin's dependency. \ No newline at end of file From 23fe6016f0c25d03bba951b0e0e13789128b5547 Mon Sep 17 00:00:00 2001 From: Findeton Date: Thu, 6 Apr 2017 18:41:34 +0200 Subject: [PATCH 68/68] change strings to int/bool --- app/models/Models.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/app/models/Models.scala b/app/models/Models.scala index 5e579927..7262b6ae 100644 --- a/app/models/Models.scala +++ b/app/models/Models.scala @@ -426,8 +426,8 @@ case class QuestionExtra( show_points: Option[Boolean], default_selected_option_ids: Option[Array[Int]], select_categories_1click: Option[Boolean], - answer_columns_size: Option[String], - group_answer_pairs: Option[String]) + answer_columns_size: Option[Int], + group_answer_pairs: Option[Boolean]) { def validate() = { @@ -451,9 +451,7 @@ case class QuestionExtra( assert(!recommended_preset__deny_text.isDefined || recommended_preset__deny_text.get.length <= LONG_STRING, "recommended_preset__deny_text too long") - assert(!answer_columns_size.isDefined || List(12,6,4,3).contains(answer_columns_size.get.toInt), "invalid answer_columns_size, can only be a string with 12,6,4,3") - - assert(!group_answer_pairs.isDefined || group_answer_pairs.get.length <= SHORT_STRING, "group_answer_pairs too long") + assert(!answer_columns_size.isDefined || List(12,6,4,3).contains(answer_columns_size.get), "invalid answer_columns_size, can only be a string with 12,6,4,3") assert(!(shuffle_all_options.isDefined && shuffle_category_list.isDefined && shuffle_all_options.get) || 0 == shuffle_category_list.get.size,