Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixing issues in deployment, including #666, #667 and #668 #670

Open
wants to merge 5 commits into
base: 3.11
Choose a base branch
from
Open

Fixing issues in deployment, including #666, #667 and #668 #670

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
588e54b
Fixing issues in deployment, including #666, #667 and #668
Glamdring Mar 19, 2018
ad61c31
Moving instanceidentity domain configuration to the IConfiguration in…
Glamdring Mar 22, 2018
98651a9
fixing prefix handling for S3 buckets
Glamdring May 5, 2018
5791842
fallback to old code in case of missing lifecycle filter
Glamdring May 5, 2018
aa44600
fixes
Glamdring Jan 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ plugins {
id 'nebula.netflixoss' version '5.0.0'
}

ext.githubProjectName = 'Priam'
ext.githubProjectName = 'Priam-parent'

allprojects {
apply plugin: 'nebula.netflixoss'
Expand Down
16 changes: 12 additions & 4 deletions priam/src/main/java/com/netflix/priam/SimpleDBConfigSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,22 @@
public final class SimpleDBConfigSource extends AbstractConfigSource {
private static final Logger logger = LoggerFactory.getLogger(SimpleDBConfigSource.class.getName());

private static final String DOMAIN = "PriamProperties";
private static String ALL_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s'";
private static final String DEFAULT_DOMAIN = "PriamProperties";
private static String ALL_QUERY = "select * from `%s` where " + Attributes.APP_ID + "='%s'";

private final Map<String, String> data = Maps.newConcurrentMap();
private final ICredential provider;

private final String domain;

@Inject
public SimpleDBConfigSource(final ICredential provider) {
this.provider = provider;
String configuredDomain = System.getProperty("priam.sdb.properties.domain");
if (configuredDomain == null) {
domain = DEFAULT_DOMAIN;
} else {
domain = configuredDomain;
}
}

@Override
Expand All @@ -68,8 +75,9 @@ public void intialize(final String asgName, final String region) {
String nextToken = null;
String appid = asgName.lastIndexOf('-') > 0 ? asgName.substring(0, asgName.indexOf('-')) : asgName;
logger.info("appid used to fetch properties is: {}", appid);
logger.info("domain used to fetch properties is: {}", domain);
do {
SelectRequest request = new SelectRequest(String.format(ALL_QUERY, appid));
SelectRequest request = new SelectRequest(String.format(ALL_QUERY, domain, appid));
request.setNextToken(nextToken);
SelectResult result = simpleDBClient.select(request);
nextToken = result.getNextToken();
Expand Down
34 changes: 25 additions & 9 deletions priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,16 @@

import java.util.*;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* DAO for handling Instance identity information such as token, zone, region
*/
@Singleton
public class SDBInstanceData {
private static final Logger logger = LoggerFactory.getLogger(SDBInstanceData.class.getName());

public static class Attributes {
public final static String APP_ID = "appId";
public final static String ID = "id";
Expand All @@ -45,17 +50,24 @@ public static class Attributes {
public final static String HOSTNAME = "hostname";
}

public static final String DOMAIN = "InstanceIdentity";
public static final String ALL_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s'";
public static final String INSTANCE_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s' and " + Attributes.LOCATION + "='%s' and " + Attributes.ID + "='%d'";
public static final String DEFAULT_DOMAIN = "InstanceIdentity";
public static final String ALL_QUERY = "select * from `%s` where " + Attributes.APP_ID + "='%s'";
public static final String INSTANCE_QUERY = "select * from `%s` where " + Attributes.APP_ID + "='%s' and " + Attributes.LOCATION + "='%s' and " + Attributes.ID + "='%d'";

private final ICredential provider;
private final IConfiguration configuration;

private final String domain;

@Inject
public SDBInstanceData(ICredential provider, IConfiguration configuration) {
this.provider = provider;
this.configuration = configuration;
String configuredDomain = System.getProperty("priam.sdb.instanceidentity.domain");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we wrap this configuration into IConfiguration?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure; will do asap

if (configuredDomain == null) {
domain = DEFAULT_DOMAIN;
} else {
domain = configuredDomain;
}
}

/**
Expand All @@ -67,7 +79,9 @@ public SDBInstanceData(ICredential provider, IConfiguration configuration) {
*/
public PriamInstance getInstance(String app, String dc, int id) {
AmazonSimpleDB simpleDBClient = getSimpleDBClient();
SelectRequest request = new SelectRequest(String.format(INSTANCE_QUERY, app, dc, id));
String query = String.format(INSTANCE_QUERY, domain, app, dc, id);
logger.info("Fetching instance data using query {}", query);
SelectRequest request = new SelectRequest(query);
SelectResult result = simpleDBClient.select(request);
if (result.getItems().size() == 0)
return null;
Expand All @@ -85,7 +99,9 @@ public Set<PriamInstance> getAllIds(String app) {
Set<PriamInstance> inslist = new HashSet<PriamInstance>();
String nextToken = null;
do {
SelectRequest request = new SelectRequest(String.format(ALL_QUERY, app));
String query = String.format(ALL_QUERY, domain, app);
logger.info("Fetching IDs using query {}", query);
SelectRequest request = new SelectRequest(query);
request.setNextToken(nextToken);
SelectResult result = simpleDBClient.select(request);
nextToken = result.getNextToken();
Expand All @@ -106,7 +122,7 @@ public Set<PriamInstance> getAllIds(String app) {
*/
public void createInstance(PriamInstance instance) throws AmazonServiceException {
AmazonSimpleDB simpleDBClient = getSimpleDBClient();
PutAttributesRequest putReq = new PutAttributesRequest(DOMAIN, getKey(instance), createAttributesToRegister(instance));
PutAttributesRequest putReq = new PutAttributesRequest(domain, getKey(instance), createAttributesToRegister(instance));
simpleDBClient.putAttributes(putReq);
}

Expand All @@ -118,7 +134,7 @@ public void createInstance(PriamInstance instance) throws AmazonServiceException
*/
public void registerInstance(PriamInstance instance) throws AmazonServiceException {
AmazonSimpleDB simpleDBClient = getSimpleDBClient();
PutAttributesRequest putReq = new PutAttributesRequest(DOMAIN, getKey(instance), createAttributesToRegister(instance));
PutAttributesRequest putReq = new PutAttributesRequest(domain, getKey(instance), createAttributesToRegister(instance));
UpdateCondition expected = new UpdateCondition();
expected.setName(Attributes.INSTANCE_ID);
expected.setExists(false);
Expand All @@ -134,7 +150,7 @@ public void registerInstance(PriamInstance instance) throws AmazonServiceExcepti
*/
public void deregisterInstance(PriamInstance instance) throws AmazonServiceException {
AmazonSimpleDB simpleDBClient = getSimpleDBClient();
DeleteAttributesRequest delReq = new DeleteAttributesRequest(DOMAIN, getKey(instance), createAttributesToDeRegister(instance));
DeleteAttributesRequest delReq = new DeleteAttributesRequest(domain, getKey(instance), createAttributesToDeRegister(instance));
simpleDBClient.deleteAttributes(delReq);
}

Expand Down
4 changes: 2 additions & 2 deletions priam/src/main/java/com/netflix/priam/defaultimpl/PriamConfigSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public PriamConfigSource(final SimpleDBConfigSource simpleDBConfigSource,
// this order was based off PriamConfigurations loading. W/e loaded last could override, but with Composite, first
// has the highest priority.
super(simpleDBConfigSource,
propertiesConfigSource,
systemPropertiesConfigSource);
systemPropertiesConfigSource,
propertiesConfigSource);
}
}
3 changes: 2 additions & 1 deletion priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.netflix.priam.aws.auth.EC2RoleAssumptionCredential;
import com.netflix.priam.aws.auth.IS3Credential;
import com.netflix.priam.aws.auth.S3RoleAssumptionCredential;
import com.netflix.priam.aws.IAMCredential;
import com.netflix.priam.backup.BackupFileSystemContext;
import com.netflix.priam.backup.IBackupFileSystem;
import com.netflix.priam.backup.IBackupMetrics;
Expand Down Expand Up @@ -63,7 +64,7 @@ protected void configure() {
bind(IFileCryptography.class).annotatedWith(Names.named("filecryptoalgorithm")).to(PgpCryptography.class);
bind(ICredentialGeneric.class).annotatedWith(Names.named("gcscredential")).to(GcsCredential.class);
bind(ICredentialGeneric.class).annotatedWith(Names.named("pgpcredential")).to(PgpCredential.class);
bind(ICredential.class).to(ClearCredential.class);
bind(ICredential.class).to(IAMCredential.class);
bind(IDeadTokenRetriever.class).to(DeadTokenRetriever.class);
bind(IPreGeneratedTokenRetriever.class).to(PreGeneratedTokenRetriever.class);
bind(INewTokenRetriever.class).to(NewTokenRetriever.class);
Expand Down
6 changes: 4 additions & 2 deletions priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,10 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed
map.put("rpc_port", config.getThriftPort());
map.put("start_native_transport", config.isNativeTransportEnabled());
map.put("native_transport_port", config.getNativeTransportPort());
map.put("listen_address", hostname);
map.put("rpc_address", hostname);
if (hostname != null) {
map.put("listen_address", hostname);
map.put("rpc_address", hostname);
}
//Dont bootstrap in restore mode
if (!Restore.isRestoreEnabled(config)) {
map.put("auto_bootstrap", config.getAutoBoostrap());
Expand Down
17 changes: 13 additions & 4 deletions priam/src/main/resources/Priam.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,27 @@ priam.backup.retention=
priam.backup.threads=2
priam.bootcluster=
priam.cache.location=/var/lib/cassandra/saved_caches
priam.cass.home=/mnt/cassandra
priam.cass.home=/etc/cassandra
priam.cass.manual.start.enable=
priam.cass.process=
priam.cass.startscript=/mnt/cassandra/bin/cassandra
priam.cass.stopscript=/mnt/cassandra/bin/cassandra
priam.cass.startscript=/etc/init.d/cassandra start
priam.cass.stopscript=/etc/init.d/cassandra stop
priam.clustername=cass_cluster
priam.commitlog.location=/var/lib/cassandra/commitlog
priam.compaction.throughput=
priam.data.location=/var/lib/cassandra/data
priam.direct.memory.size.m1.large=1G
priam.endpoint_snitch=org.apache.cassandra.locator.Ec2Snitch
priam.heap.newgen.size.m1.large=2G
priam.heap.size.m1.large=4G
priam.heap.size.m1.large=1G
priam.heap.newgen.size.t2.small=1G
priam.heap.size.t2.small=1G
priam.heap.newgen.size.t2.micro=512M
priam.heap.size.t2.micro=512M
priam.heap.newgen.size.m3.medium=1G
priam.heap.size.m3.medium=1G
priam.heap.newgen.size.m3.large=1G
priam.heap.size.m3.large=1G
priam.hint.delay=
priam.hint.window=
priam.jmx.port=7199
Expand Down Expand Up @@ -53,3 +61,4 @@ priam.thrift.port=9160
priam.upload.throttle=
priam.yamlLocation=
priam.zones.available=
priam.nativeTransport.enabled=true