From cd920c8a87716b717be4246bd75b1fc008affd8b Mon Sep 17 00:00:00 2001 From: Eamonn Postlethwaite Date: Tue, 18 Jan 2022 16:52:48 +0000 Subject: [PATCH 1/4] first draft of continuing BKZ even if solution seems possible --- lwe_challenge.py | 94 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 18 deletions(-) diff --git a/lwe_challenge.py b/lwe_challenge.py index df35169..1d7ee8f 100755 --- a/lwe_challenge.py +++ b/lwe_challenge.py @@ -14,6 +14,7 @@ from fpylll import BKZ as fplll_bkz from fpylll.algorithms.bkz2 import BKZReduction +from fpylll.tools.bkz_simulator import simulate from fpylll.tools.quality import basis_quality from fpylll.util import gaussian_heuristic, set_threads @@ -153,10 +154,11 @@ def lwe_kernel(arg0, params=None, seed=None): T0 = time.time() T0_BKZ = time.time() + T_blocksizes = {} for blocksize in blocksizes: for tt in range(tours): # BKZ tours - + T0_this_blocksize = time.time() if blocksize < fpylll_crossover: if verbose: print( "Starting a fpylll BKZ-%d tour. " % (blocksize), end='') @@ -178,8 +180,12 @@ def lwe_kernel(arg0, params=None, seed=None): goal_r0=target_norm, pump_params=pump_params) + # total time spent on BKZ thus far T_BKZ = time.time() - T0_BKZ + # time for one tour of this blocksize + T_blocksizes[blocksize] = time.time() - T0_this_blocksize + if verbose: slope = basis_quality(g6k.M)["/"] fmt = "slope: %.5f, walltime: %.3f sec" @@ -190,24 +196,76 @@ def lwe_kernel(arg0, params=None, seed=None): if g6k.M.get_r(0, 0) <= target_norm: break - # overdoing n_max would allocate too much memory, so we are careful - svp_Tmax = svp_bkz_time_factor * T_BKZ expo = 0.292 - # solving for maximal d s.t. 2^{expo * d} / param.threads <= svp_Tmax - # cannot figure out the additive 58, will leave for now - n_max = int(58+6 + (1./expo) * log(svp_Tmax * params.threads)/log(2.)) - - rr = [g6k.M.get_r(i, i) for i in range(d)] - for n_expected in range(2, d-2): - x = (target_norm/goal_margin) * n_expected/(1.*d) - if 4./3 * gaussian_heuristic(rr[d-n_expected:]) > x: - break - - print( "Without otf, would expect solution at pump-%d. n_max=%d in the given time." % (n_expected, n_max) )# noqa - if n_expected >= n_max - 1: + + def attainable_SVP_dim(T_BKZ): + svp_Tmax = svp_bkz_time_factor * T_BKZ + # solving for maximal d s.t. 2^{expo * d} / param.threads <= svp_Tmax + return int(58+6 + (1./expo) * log(svp_Tmax * params.threads)/log(2.)) + + def expected_successful_SVP_dim(rr=None): + if rr is None: + rr = [g6k.M.get_r(i, i) for i in range(d)] + for n_expected in range(2, d): + x = (target_norm/goal_margin) * n_expected/(1.*d) + if 4./3 * gaussian_heuristic(rr[d-n_expected:]) > x: + break + return n_expected + + # given current state of basis + current_n_max = attainable_SVP_dim(T_BKZ) + current_n_expected = expected_successful_SVP_dim() + + print("Without otf, would expect solution at pump-%d. n_max=%d in the given time." % (current_n_expected, current_n_max) )# noqa + if current_n_expected >= current_n_max - 1: + # stronger BKZ required + continue + + # perhaps we still prefer stronger BKZ, e.g. to reduce memory + # currently look at the next three potential tours + ind = blocksizes.index(blocksize) + remaining_blocksizes = blocksizes[ind+1:ind+3] + + # expected extra BKZ times, and future basis profiles + extra_T_BKZs = [] + future_rrs = [] + + # current data on blocksize, basis profile, and individual blocksize times + previous_blocksize = blocksize + rr = list(g6k.M.r()) + extra_T_BKZ = 0 + temp_T_blocksizes = T_blocksizes + + for rblock in remaining_blocksizes: + # simulate basis profile after next tour, and update basis profile + params = fplll_bkz.Param(block_size=rblock, max_loops=1) + rr, _ = simulate(rr, params) + future_rrs += [rr] + + # time for BKZ-rblock tour calculated as a function of the previous tour + T_rblock_BKZ = temp_T_blocksizes[previous_blocksize] * 2**(expo * (rblock - previous_blocksize)) + temp_T_blocksizes[rblock] = T_rblock_BKZ + + # total estimated BKZ time with rblock tour added and update previous blocksize + extra_T_BKZ += T_rblock_BKZ + extra_T_BKZs += [extra_T_BKZ] + previous_blocksize = rblock + + more_BKZ = False + + for (rr, extra_T_BKZ) in zip(future_rrs, extra_T_BKZs): + potential_n_expected = expected_successful_SVP_dim(rr=rr) + svp_dim_reduction = potential_n_expected - current_n_expected + if svp_dim_reduction >= 0: + continue + time_SVP_current = 2**(expo * (current_n_expected - 64)) / params.threads + if extra_T_BKZ < (1. - 2**(expo*svp_dim_reduction)) * time_SVP_current: + more_BKZ = True + + if more_BKZ: continue - n_max += 1 + current_n_max += 1 # Larger SVP @@ -215,9 +273,9 @@ def lwe_kernel(arg0, params=None, seed=None): while gaussian_heuristic([g6k.M.get_r(i, i) for i in range(llb, d)]) < target_norm * (d - llb)/(1.*d): # noqa llb -= 1 - f = d-llb-n_max + f = d-llb-current_n_max if verbose: - print( "Starting svp pump_{%d, %d, %d}, n_max = %d, Tmax= %.2f sec" % (llb, d-llb, f, n_max, svp_Tmax) ) # noqa + print( "Starting svp pump_{%d, %d, %d}, n_max = %d, Tmax= %.2f sec" % (llb, d-llb, f, current_n_max, svp_Tmax) ) # noqa pump(g6k, tracer, llb, d-llb, f, verbose=verbose, goal_r0=target_norm * (d - llb)/(1.*d)) From 0cdd4d0890ecefd2033ad1c9935beb4c3265f041 Mon Sep 17 00:00:00 2001 From: Eamonn Postlethwaite Date: Tue, 18 Jan 2022 18:03:40 +0100 Subject: [PATCH 2/4] some flake8 cleaning --- lwe_challenge.py | 52 +++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/lwe_challenge.py b/lwe_challenge.py index 1d7ee8f..902b130 100755 --- a/lwe_challenge.py +++ b/lwe_challenge.py @@ -109,8 +109,8 @@ def lwe_kernel(arg0, params=None, seed=None): set_threads(threads) A, c, q = load_lwe_challenge(n=n, alpha=alpha) - print( "-------------------------" ) - print( "Primal attack, LWE challenge n=%d, alpha=%.4f" % (n, alpha) ) + print("-------------------------") + print("Primal attack, LWE challenge n=%d, alpha=%.4f" % (n, alpha)) if m is None: try: @@ -126,7 +126,7 @@ def lwe_kernel(arg0, params=None, seed=None): (b, s, _) = min_cost_param except TypeError: raise TypeError("No winning parameters.") - print( "Chose %d samples. Predict solution at bkz-%d + svp-%d" % (m, b, s)) + print("Chose %d samples. Predict solution at bkz-%d + svp-%d" % (m, b, s)) print("") target_norm = goal_margin * (alpha*q)**2 * m + 1 @@ -134,13 +134,12 @@ def lwe_kernel(arg0, params=None, seed=None): if blocksizes is not None: blocksizes = list(range(10, 40)) + eval("range(%s)" % re.sub(":", ",", blocksizes)) # noqa else: -# blocksizes = list(range(10, 50)) + list(range(90, b-15, 5)) + [b-15] + list(range(b - 13, b + 25, 2)) - blocksizes = list(range(10, 50)) + list(range(70, b-15, 10)) + [b-15] + list(range(b - 13, b + 25, 2)) + blocksizes = list(range(10, 50)) + list(range(70, b-15, 10)) + [b-15] + list(range(b - 13, b + 25, 2)) # noqa B = primal_lattice_basis(A, c, q, m=m) g6k = Siever(B, params) - print( "GSO precision: ", g6k.M.float_type ) + print("GSO precision: ", g6k.M.float_type) if dont_trace: tracer = dummy_tracer @@ -150,7 +149,7 @@ def lwe_kernel(arg0, params=None, seed=None): d = g6k.full_n g6k.lll(0, g6k.full_n) slope = basis_quality(g6k.M)["/"] - print( "Intial Slope = %.5f\n" % slope ) + print("Intial Slope = %.5f\n" % slope) T0 = time.time() T0_BKZ = time.time() @@ -161,7 +160,8 @@ def lwe_kernel(arg0, params=None, seed=None): T0_this_blocksize = time.time() if blocksize < fpylll_crossover: if verbose: - print( "Starting a fpylll BKZ-%d tour. " % (blocksize), end='') + print("Starting a fpylll BKZ-%d tour. " % + (blocksize), end='') sys.stdout.flush() bkz = BKZReduction(g6k.M) par = fplll_bkz.Param(blocksize, @@ -171,7 +171,7 @@ def lwe_kernel(arg0, params=None, seed=None): else: if verbose: - print( "Starting a pnjBKZ-%d tour. " % (blocksize) ) + print("Starting a pnjBKZ-%d tour. " % (blocksize)) pump_n_jump_bkz_tour(g6k, tracer, blocksize, jump=jump, verbose=verbose, @@ -189,7 +189,7 @@ def lwe_kernel(arg0, params=None, seed=None): if verbose: slope = basis_quality(g6k.M)["/"] fmt = "slope: %.5f, walltime: %.3f sec" - print( fmt % (slope, time.time() - T0) ) + print(fmt % (slope, time.time() - T0)) g6k.lll(0, g6k.full_n) @@ -200,8 +200,9 @@ def lwe_kernel(arg0, params=None, seed=None): def attainable_SVP_dim(T_BKZ): svp_Tmax = svp_bkz_time_factor * T_BKZ - # solving for maximal d s.t. 2^{expo * d} / param.threads <= svp_Tmax - return int(58+6 + (1./expo) * log(svp_Tmax * params.threads)/log(2.)) + # solve max d s.t. 2^{expo * d} / param.threads <= svp_Tmax + return int(58+6 + (1./expo) * + log(svp_Tmax * params.threads)/log(2.)) def expected_successful_SVP_dim(rr=None): if rr is None: @@ -216,7 +217,7 @@ def expected_successful_SVP_dim(rr=None): current_n_max = attainable_SVP_dim(T_BKZ) current_n_expected = expected_successful_SVP_dim() - print("Without otf, would expect solution at pump-%d. n_max=%d in the given time." % (current_n_expected, current_n_max) )# noqa + print("Without otf, would expect solution at pump-%d. n_max=%d in the given time." % (current_n_expected, current_n_max)) # noqa if current_n_expected >= current_n_max - 1: # stronger BKZ required continue @@ -230,23 +231,23 @@ def expected_successful_SVP_dim(rr=None): extra_T_BKZs = [] future_rrs = [] - # current data on blocksize, basis profile, and individual blocksize times + # data on blocksize, basis profile, individual blocksize times previous_blocksize = blocksize rr = list(g6k.M.r()) extra_T_BKZ = 0 temp_T_blocksizes = T_blocksizes for rblock in remaining_blocksizes: - # simulate basis profile after next tour, and update basis profile + # simulate basis profile after next tour, update basis profile params = fplll_bkz.Param(block_size=rblock, max_loops=1) rr, _ = simulate(rr, params) future_rrs += [rr] - # time for BKZ-rblock tour calculated as a function of the previous tour - T_rblock_BKZ = temp_T_blocksizes[previous_blocksize] * 2**(expo * (rblock - previous_blocksize)) + # time for BKZ-rblock tour as a function of the previous tour + T_rblock_BKZ = temp_T_blocksizes[previous_blocksize] * 2**(expo * (rblock - previous_blocksize)) # noqa temp_T_blocksizes[rblock] = T_rblock_BKZ - # total estimated BKZ time with rblock tour added and update previous blocksize + # total extra BKZ time with rblock tour extra_T_BKZ += T_rblock_BKZ extra_T_BKZs += [extra_T_BKZ] previous_blocksize = rblock @@ -255,11 +256,12 @@ def expected_successful_SVP_dim(rr=None): for (rr, extra_T_BKZ) in zip(future_rrs, extra_T_BKZs): potential_n_expected = expected_successful_SVP_dim(rr=rr) - svp_dim_reduction = potential_n_expected - current_n_expected - if svp_dim_reduction >= 0: + svp_dim_dec = potential_n_expected - current_n_expected + if svp_dim_dec >= 0: continue - time_SVP_current = 2**(expo * (current_n_expected - 64)) / params.threads - if extra_T_BKZ < (1. - 2**(expo*svp_dim_reduction)) * time_SVP_current: + time_SVP_current = 2**(expo * (current_n_expected - 64)) + time_SVP_current /= params.threads + if extra_T_BKZ < (1-2**(expo*svp_dim_dec)) * time_SVP_current: more_BKZ = True if more_BKZ: @@ -282,7 +284,7 @@ def expected_successful_SVP_dim(rr=None): if verbose: slope = basis_quality(g6k.M)["/"] fmt = "\n slope: %.5f, walltime: %.3f sec" - print( fmt % (slope, time.time() - T0) ) + print(fmt % (slope, time.time() - T0)) print g6k.lll(0, g6k.full_n) @@ -291,8 +293,8 @@ def expected_successful_SVP_dim(rr=None): break if g6k.M.get_r(0, 0) <= target_norm: - print( "Finished! TT=%.2f sec" % (time.time() - T0) ) - print( g6k.M.B[0] ) + print("Finished! TT=%.2f sec" % (time.time() - T0)) + print(g6k.M.B[0]) alpha_ = int(alpha*1000) filename = 'lwechallenge/%03d-%03d-solution.txt' % (n, alpha_) fn = open(filename, "w") From 5789bc6802509cf22e448b02ba2697ab2f95da7a Mon Sep 17 00:00:00 2001 From: Eamonn Postlethwaite Date: Tue, 18 Jan 2022 22:49:26 +0000 Subject: [PATCH 3/4] some bug fixes and a factor to tune memory importance --- lwe_challenge.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lwe_challenge.py b/lwe_challenge.py index 902b130..e8f88bb 100755 --- a/lwe_challenge.py +++ b/lwe_challenge.py @@ -94,6 +94,7 @@ def lwe_kernel(arg0, params=None, seed=None): # flow of the lwe solver svp_bkz_time_factor = params.pop("lwe/svp_bkz_time_factor") goal_margin = params.pop("lwe/goal_margin") + favour_memory_factor = params.pop("lwe/favour_memory_factor") # generation of lwe instance and Kannan's embedding alpha = params.pop("lwe/alpha") @@ -197,9 +198,9 @@ def lwe_kernel(arg0, params=None, seed=None): break expo = 0.292 + svp_Tmax = svp_bkz_time_factor * T_BKZ def attainable_SVP_dim(T_BKZ): - svp_Tmax = svp_bkz_time_factor * T_BKZ # solve max d s.t. 2^{expo * d} / param.threads <= svp_Tmax return int(58+6 + (1./expo) * log(svp_Tmax * params.threads)/log(2.)) @@ -225,7 +226,7 @@ def expected_successful_SVP_dim(rr=None): # perhaps we still prefer stronger BKZ, e.g. to reduce memory # currently look at the next three potential tours ind = blocksizes.index(blocksize) - remaining_blocksizes = blocksizes[ind+1:ind+3] + remaining_blocksizes = blocksizes[ind+1:ind+4] # expected extra BKZ times, and future basis profiles extra_T_BKZs = [] @@ -239,8 +240,8 @@ def expected_successful_SVP_dim(rr=None): for rblock in remaining_blocksizes: # simulate basis profile after next tour, update basis profile - params = fplll_bkz.Param(block_size=rblock, max_loops=1) - rr, _ = simulate(rr, params) + bkz_params = fplll_bkz.Param(block_size=rblock, max_loops=1) + rr, _ = simulate(rr, bkz_params) future_rrs += [rr] # time for BKZ-rblock tour as a function of the previous tour @@ -261,7 +262,7 @@ def expected_successful_SVP_dim(rr=None): continue time_SVP_current = 2**(expo * (current_n_expected - 64)) time_SVP_current /= params.threads - if extra_T_BKZ < (1-2**(expo*svp_dim_dec)) * time_SVP_current: + if extra_T_BKZ < favour_memory_factor * (1-2**(expo*svp_dim_dec)) * time_SVP_current: # noqa more_BKZ = True if more_BKZ: @@ -317,6 +318,7 @@ def lwe(): lwe__m=None, lwe__goal_margin=1.5, lwe__svp_bkz_time_factor=1, + lwe__favour_memory_factor=1, bkz__blocksizes=None, bkz__tours=1, bkz__jump=1, From 728fb8a88d37b8e579bbebeaf771a04b10f64ff8 Mon Sep 17 00:00:00 2001 From: Eamonn Postlethwaite Date: Tue, 18 Jan 2022 23:12:49 +0000 Subject: [PATCH 4/4] accidentally overriding svp_bkz_time_factor --- lwe_challenge.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lwe_challenge.py b/lwe_challenge.py index e8f88bb..ea29e34 100755 --- a/lwe_challenge.py +++ b/lwe_challenge.py @@ -200,10 +200,10 @@ def lwe_kernel(arg0, params=None, seed=None): expo = 0.292 svp_Tmax = svp_bkz_time_factor * T_BKZ - def attainable_SVP_dim(T_BKZ): + def attainable_SVP_dim(T_svp): # solve max d s.t. 2^{expo * d} / param.threads <= svp_Tmax return int(58+6 + (1./expo) * - log(svp_Tmax * params.threads)/log(2.)) + log(T_svp * params.threads)/log(2.)) def expected_successful_SVP_dim(rr=None): if rr is None: @@ -215,7 +215,7 @@ def expected_successful_SVP_dim(rr=None): return n_expected # given current state of basis - current_n_max = attainable_SVP_dim(T_BKZ) + current_n_max = attainable_SVP_dim(svp_Tmax) current_n_expected = expected_successful_SVP_dim() print("Without otf, would expect solution at pump-%d. n_max=%d in the given time." % (current_n_expected, current_n_max)) # noqa