[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] r32161 - gnunet/src/ats
From: |
gnunet |
Subject: |
[GNUnet-SVN] r32161 - gnunet/src/ats |
Date: |
Mon, 3 Feb 2014 14:52:04 +0100 |
Author: oehlmann
Date: 2014-02-03 14:52:04 +0100 (Mon, 03 Feb 2014)
New Revision: 32161
Modified:
gnunet/src/ats/plugin_ats_ril.c
Log:
-minor renamings
-default parameters set to better values
-exploration decay added
Modified: gnunet/src/ats/plugin_ats_ril.c
===================================================================
--- gnunet/src/ats/plugin_ats_ril.c 2014-02-03 12:50:52 UTC (rev 32160)
+++ gnunet/src/ats/plugin_ats_ril.c 2014-02-03 13:52:04 UTC (rev 32161)
@@ -28,26 +28,27 @@
#define LOG(kind,...) GNUNET_log_from (kind, "ats-ril",__VA_ARGS__)
-#define RIL_MIN_BW (1 * ntohl
(GNUNET_CONSTANTS_DEFAULT_BW_IN_OUT.value__))
+#define RIL_MIN_BW (5 * ntohl
(GNUNET_CONSTANTS_DEFAULT_BW_IN_OUT.value__))
#define RIL_MAX_BW GNUNET_ATS_MaxBandwidth
#define RIL_ACTION_INVALID -1
#define RIL_INTERVAL_EXPONENT 10
#define RIL_UTILITY_DELAY_MAX 1000
-#define RIL_DEFAULT_STEP_TIME_MIN GNUNET_TIME_relative_multiply
(GNUNET_TIME_UNIT_MILLISECONDS, 200)
-#define RIL_DEFAULT_STEP_TIME_MAX GNUNET_TIME_relative_multiply
(GNUNET_TIME_UNIT_MILLISECONDS, 1000)
+#define RIL_DEFAULT_STEP_TIME_MIN GNUNET_TIME_relative_multiply
(GNUNET_TIME_UNIT_MILLISECONDS, 300)
+#define RIL_DEFAULT_STEP_TIME_MAX GNUNET_TIME_relative_multiply
(GNUNET_TIME_UNIT_MILLISECONDS, 2000)
#define RIL_DEFAULT_ALGORITHM RIL_ALGO_Q
#define RIL_DEFAULT_SELECT RIL_SELECT_EGREEDY
-#define RIL_DEFAULT_WELFARE RIL_WELFARE_EGALITARIAN
-#define RIL_DEFAULT_DISCOUNT_BETA 1.0
+#define RIL_DEFAULT_WELFARE RIL_WELFARE_NASH
+#define RIL_DEFAULT_DISCOUNT_BETA 2.0
#define RIL_DEFAULT_DISCOUNT_GAMMA 0.5
-#define RIL_DEFAULT_GRADIENT_STEP_SIZE 0.1
+#define RIL_DEFAULT_GRADIENT_STEP_SIZE 0.001
#define RIL_DEFAULT_TRACE_DECAY 0.5
-#define RIL_DEFAULT_EXPLORE_RATIO 0.1
+#define RIL_DEFAULT_EXPLORE_RATIO 1
+#define RIL_DEFAULT_EXPLORE_DECAY 0.95
#define RIL_DEFAULT_RBF_DIVISOR 50
-#define RIL_DEFAULT_GLOBAL_REWARD_SHARE 0.5
-#define RIL_DEFAULT_TEMPERATURE 1.0
+#define RIL_DEFAULT_TEMPERATURE 10
+#define RIL_DEFAULT_TEMPERATURE_DECAY 0.999
#define RIL_INC_DEC_STEP_SIZE 1
#define RIL_NOP_DECAY 0.5
@@ -140,11 +141,21 @@
enum RIL_E_Modification eligibility_trace_mode;
/**
+ * Initial softmax action-selection temperature
+ */
+ double temperature_init;
+
+ /**
* Softmax action-selection temperature
*/
double temperature;
/**
+ * Decay factor of the temperature value
+ */
+ double temperature_decay;
+
+ /**
* Which measure of social welfare should be used
*/
enum RIL_Welfare social_welfare;
@@ -160,14 +171,19 @@
enum RIL_Select select;
/**
+ * Initial exploration ratio value
+ */
+ double explore_ratio_init;
+
+ /**
* Ratio, with what probability an agent should explore in the e-greed policy
*/
double explore_ratio;
/**
- * How big the share of the global part of the reward signal is
+ * Decay factor of the explore ratio
*/
- double reward_global_share;
+ double explore_ratio_decay;
/**
* Minimal interval time between steps in milliseconds
@@ -1044,7 +1060,6 @@
double objective;
double delta;
double steady;
- double pen_share;
double penalty;
double reward;
@@ -1065,11 +1080,10 @@
steady = (RIL_ACTION_NOTHING == agent->a_old) ? agent->nop_bonus : 0;
- pen_share = 0.5;
penalty = envi_get_penalty(solver, agent);
reward = delta + steady;
- return ((1 - pen_share) * reward) + (pen_share * penalty);
+ return reward + penalty;
}
/**
@@ -1316,6 +1330,7 @@
{
agent->eligibility_reset = GNUNET_YES;
}
+ agent->envi->parameters.explore_ratio *=
agent->envi->parameters.explore_ratio_decay;
return action;
}
else //exploit
@@ -1373,10 +1388,11 @@
{
if (sum + p[i] > r)
{
- if (RIL_ALGO_Q == agent->envi->parameters.algorithm)
+ if (i != a_max)
{
- if (i != a_max)
+ if (RIL_ALGO_Q == agent->envi->parameters.algorithm)
agent->eligibility_reset = GNUNET_YES;
+ agent->envi->parameters.temperature *=
agent->envi->parameters.temperature_decay;
}
return i;
}
@@ -1643,7 +1659,7 @@
struct GNUNET_TIME_Relative time_delta;
double tau;
- // MDP case - TODO! remove when debugged and test SMDP case
+ // MDP case only for debugging purposes
if (solver->simulate)
{
solver->global_discount_variable = solver->parameters.gamma;
@@ -2099,7 +2115,11 @@
"API_address_change_preference() Preference '%s' for peer '%s' changed
to %.2f \n",
GNUNET_ATS_print_preference_type (kind), GNUNET_i2s (peer), pref_rel);
- ril_step (solver);
+ struct GAS_RIL_Handle *s = solver;
+
+ s->parameters.temperature = s->parameters.temperature_init;
+ s->parameters.explore_ratio = s->parameters.explore_ratio_init;
+ ril_step (s);
}
/**
@@ -2128,18 +2148,21 @@
{
solver->parameters.rbf_divisor = RIL_DEFAULT_RBF_DIVISOR;
}
+
if (GNUNET_OK
!= GNUNET_CONFIGURATION_get_value_time (env->cfg, "ats",
"RIL_STEP_TIME_MIN",
&solver->parameters.step_time_min))
{
solver->parameters.step_time_min = RIL_DEFAULT_STEP_TIME_MIN;
}
+
if (GNUNET_OK
!= GNUNET_CONFIGURATION_get_value_time (env->cfg, "ats",
"RIL_STEP_TIME_MAX",
&solver->parameters.step_time_max))
{
solver->parameters.step_time_max = RIL_DEFAULT_STEP_TIME_MAX;
}
+
if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats",
"RIL_ALGORITHM", &string))
{
solver->parameters.algorithm = !strcmp (string, "SARSA") ? RIL_ALGO_SARSA
: RIL_ALGO_Q;
@@ -2149,6 +2172,7 @@
{
solver->parameters.algorithm = RIL_DEFAULT_ALGORITHM;
}
+
if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats",
"RIL_SELECT", &string))
{
solver->parameters.select = !strcmp (string, "EGREEDY") ?
RIL_SELECT_EGREEDY : RIL_SELECT_SOFTMAX;
@@ -2158,6 +2182,7 @@
{
solver->parameters.select = RIL_DEFAULT_SELECT;
}
+
if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats",
"RIL_DISCOUNT_BETA", &string))
{
solver->parameters.beta = strtod (string, NULL);
@@ -2172,6 +2197,7 @@
{
solver->parameters.beta = RIL_DEFAULT_DISCOUNT_BETA;
}
+
if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats",
"RIL_DISCOUNT_GAMMA", &string))
{
solver->parameters.gamma = strtod (string, NULL);
@@ -2186,6 +2212,7 @@
{
solver->parameters.gamma = RIL_DEFAULT_DISCOUNT_GAMMA;
}
+
if (GNUNET_OK
== GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats",
"RIL_GRADIENT_STEP_SIZE", &string))
{
@@ -2201,6 +2228,7 @@
{
solver->parameters.alpha = RIL_DEFAULT_GRADIENT_STEP_SIZE;
}
+
if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats",
"RIL_TRACE_DECAY", &string))
{
solver->parameters.lambda = strtod (string, NULL);
@@ -2215,52 +2243,72 @@
{
solver->parameters.lambda = RIL_DEFAULT_TRACE_DECAY;
}
+
if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats",
"RIL_EXPLORE_RATIO", &string))
{
- solver->parameters.explore_ratio = strtod (string, NULL);
+ solver->parameters.explore_ratio_init = strtod (string, NULL);
GNUNET_free (string);
- if (solver->parameters.explore_ratio < 0 ||
solver->parameters.explore_ratio > 1)
+ if (solver->parameters.explore_ratio_init < 0 ||
solver->parameters.explore_ratio_init > 1)
{
LOG (GNUNET_ERROR_TYPE_WARNING, "RIL_EXPLORE_RATIO not configured in
range [0,1]. Set to default value of %f instead.\n", RIL_DEFAULT_EXPLORE_RATIO);
- solver->parameters.explore_ratio = RIL_DEFAULT_EXPLORE_RATIO;
+ solver->parameters.explore_ratio_init = RIL_DEFAULT_EXPLORE_RATIO;
}
}
else
{
- solver->parameters.explore_ratio = RIL_DEFAULT_EXPLORE_RATIO;
+ solver->parameters.explore_ratio_init = RIL_DEFAULT_EXPLORE_RATIO;
}
- if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats",
"RIL_GLOBAL_REWARD_SHARE", &string))
+
+ if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats",
"RIL_EXPLORE_DECAY", &string))
{
- solver->parameters.reward_global_share = strtod (string, NULL);
+ solver->parameters.explore_ratio_decay = strtod (string, NULL);
GNUNET_free (string);
- if (solver->parameters.reward_global_share < 0 ||
solver->parameters.reward_global_share > 1)
+ if (solver->parameters.explore_ratio_decay < 0 ||
solver->parameters.explore_ratio_decay > 1)
{
- LOG (GNUNET_ERROR_TYPE_WARNING, "RIL_GLOBAL_REWARD_SHARE not configured
in range [0,1]. Set to default value of %f instead.\n",
RIL_DEFAULT_GLOBAL_REWARD_SHARE);
- solver->parameters.reward_global_share = RIL_DEFAULT_GLOBAL_REWARD_SHARE;
+ LOG (GNUNET_ERROR_TYPE_WARNING, "RIL_EXPLORE_RATIO not configured in
range [0,1]. Set to default value of %f instead.\n", RIL_DEFAULT_EXPLORE_DECAY);
+ solver->parameters.explore_ratio_decay = RIL_DEFAULT_EXPLORE_DECAY;
}
}
else
{
- solver->parameters.reward_global_share = RIL_DEFAULT_GLOBAL_REWARD_SHARE;
+ solver->parameters.explore_ratio_decay = RIL_DEFAULT_EXPLORE_DECAY;
}
+
if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats",
"RIL_TEMPERATURE", &string))
{
- solver->parameters.temperature = strtod (string, NULL);
+ solver->parameters.temperature_init = strtod (string, NULL);
GNUNET_free (string);
- if (solver->parameters.temperature <= 0)
+ if (solver->parameters.temperature_init <= 0)
{
LOG (GNUNET_ERROR_TYPE_WARNING, "RIL_TEMPERATURE not positive. Set to
default value of %f instead.\n", RIL_DEFAULT_TEMPERATURE);
- solver->parameters.temperature = RIL_DEFAULT_TEMPERATURE;
+ solver->parameters.temperature_init = RIL_DEFAULT_TEMPERATURE;
}
}
else
{
- solver->parameters.temperature = RIL_DEFAULT_TEMPERATURE;
+ solver->parameters.temperature_init = RIL_DEFAULT_TEMPERATURE;
}
+
+ if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats",
"RIL_TEMPERATURE_DECAY", &string))
+ {
+ solver->parameters.temperature_decay = strtod (string, NULL);
+ GNUNET_free (string);
+ if (solver->parameters.temperature_decay <= 0 ||
solver->parameters.temperature_decay > 1)
+ {
+ LOG (GNUNET_ERROR_TYPE_WARNING, "RIL_TEMPERATURE_DECAY not in range
[0,1]. Set to default value of %f instead.\n", RIL_DEFAULT_TEMPERATURE_DECAY);
+ solver->parameters.temperature_decay = RIL_DEFAULT_TEMPERATURE_DECAY;
+ }
+ }
+ else
+ {
+ solver->parameters.temperature_decay = RIL_DEFAULT_TEMPERATURE_DECAY;
+ }
+
if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (env->cfg, "ats",
"RIL_SIMULATE", &solver->simulate))
{
solver->simulate = GNUNET_NO;
}
+
if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno(env->cfg, "ats",
"RIL_REPLACE_TRACES"))
{
solver->parameters.eligibility_trace_mode = RIL_E_REPLACE;
@@ -2269,6 +2317,7 @@
{
solver->parameters.eligibility_trace_mode = RIL_E_ACCUMULATE;
}
+
if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats",
"RIL_SOCIAL_WELFARE", &string))
{
solver->parameters.social_welfare = !strcmp (string, "NASH") ?
RIL_WELFARE_NASH : RIL_WELFARE_EGALITARIAN;
@@ -2313,11 +2362,10 @@
solver->parameters.alpha,
solver->parameters.beta,
solver->parameters.lambda);
- LOG(GNUNET_ERROR_TYPE_DEBUG, "init() exploration_ratio = %f, temperature =
%f, ActionSelection = %s, global_share = %f\n",
+ LOG(GNUNET_ERROR_TYPE_DEBUG, "init() exploration_ratio = %f, temperature =
%f, ActionSelection = %s\n",
solver->parameters.explore_ratio,
solver->parameters.temperature,
- solver->parameters.select ? "EGREEDY" : "SOFTMAX",
- solver->parameters.reward_global_share);
+ solver->parameters.select ? "EGREEDY" : "SOFTMAX");
LOG(GNUNET_ERROR_TYPE_DEBUG, "init() RBF_DIVISOR = %llu\n",
solver->parameters.rbf_divisor);
@@ -2395,6 +2443,9 @@
return;
}
+ s->parameters.temperature = s->parameters.temperature_init;
+ s->parameters.explore_ratio = s->parameters.explore_ratio_init;
+
agent = ril_get_agent (s, &address->peer, GNUNET_YES);
//add address
@@ -2478,6 +2529,9 @@
return;
}
+ s->parameters.temperature = s->parameters.temperature_init;
+ s->parameters.explore_ratio = s->parameters.explore_ratio_init;
+
address_index = agent_address_get_index (agent, address);
address_wrapped = agent_address_get_wrapped (agent, address);
@@ -2564,7 +2618,11 @@
"to %.2f \n", GNUNET_ATS_print_property_type (type), GNUNET_i2s
(&address->peer),
address->addr, rel_value);
- ril_step (solver);
+ struct GAS_RIL_Handle *s = solver;
+
+ s->parameters.temperature = s->parameters.temperature_init;
+ s->parameters.explore_ratio = s->parameters.explore_ratio_init;
+ ril_step (s);
}
/**
@@ -2628,6 +2686,9 @@
(GNUNET_YES == address->active) ? "active" : "inactive", GNUNET_i2s
(&address->peer),
GNUNET_ATS_print_network_type (current_network),
GNUNET_ATS_print_network_type (new_network));
+ s->parameters.temperature = s->parameters.temperature_init;
+ s->parameters.explore_ratio = s->parameters.explore_ratio_init;
+
if (address->active && !ril_network_is_active (solver, new_network))
{
GAS_ril_address_delete (solver, address, GNUNET_NO);
@@ -2724,9 +2785,6 @@
const struct ATS_Address *
GAS_ril_get_preferred_address (void *solver, const struct GNUNET_PeerIdentity
*peer)
{
- /*
- * activate agent, return currently chosen address
- */
struct GAS_RIL_Handle *s = solver;
struct RIL_Peer_Agent *agent;
@@ -2750,6 +2808,8 @@
LOG(GNUNET_ERROR_TYPE_DEBUG,
"API_get_preferred_address() Activated agent for peer '%s', but no
address available\n",
GNUNET_i2s (peer));
+ s->parameters.temperature = s->parameters.temperature_init;
+ s->parameters.explore_ratio = s->parameters.explore_ratio_init;
}
return agent->address_inuse;
}
@@ -2784,6 +2844,9 @@
return;
}
+ s->parameters.temperature = s->parameters.temperature_init;
+ s->parameters.explore_ratio = s->parameters.explore_ratio_init;
+
agent->is_active = GNUNET_NO;
envi_set_active_suggestion (s, agent, agent->address_inuse, agent->bw_in,
agent->bw_out,
[Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] r32161 - gnunet/src/ats,
gnunet <=