Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix entity too large #658

Open
wants to merge 22 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,12 @@ class BuildConfig {
@Value('${wave.build.retry-attempts:0}')
int retryAttempts

@Value('${wave.build.max-conda-file-size:100000}')
int maxCondaFileSize

@Value('${wave.build.max-container-file-size:100000}')
int maxContainerFileSize

@PostConstruct
private void init() {
log.info("Builder config: " +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -278,8 +278,8 @@ class ContainerController {

protected void storeContainerRequest0(SubmitContainerTokenRequest req, ContainerRequestData data, TokenData token, String target, String ip) {
try {
final recrd = new WaveContainerRecord(req, data, target, ip, token.expiration)
persistenceService.saveContainerRequest(token.value, recrd)
final recrd = new WaveContainerRecord(req, data, token.value, target, ip, token.expiration)
persistenceService.saveContainerRequest(recrd)
}
catch (Throwable e) {
log.error("Unable to store container request with token: ${token}", e)
Expand Down Expand Up @@ -522,6 +522,12 @@ class ContainerController {

void validateContainerRequest(SubmitContainerTokenRequest req) throws BadRequestException {
String msg
//check conda file size
if( req.condaFile && req.condaFile.length() > buildConfig.maxCondaFileSize )
throw new BadRequestException("Conda file size exceeds the maximum allowed size of ${buildConfig.maxCondaFileSize} bytes")
// check container file size
if( req.containerFile && req.containerFile.length() > buildConfig.maxContainerFileSize )
throw new BadRequestException("Container file size exceeds the maximum allowed size of ${buildConfig.maxContainerFileSize} bytes")
// check valid image name
msg = validationService.checkContainerName(req.containerImage)
if( msg ) throw new BadRequestException(msg)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,9 @@ interface PersistenceService {
/**
* Store a {@link WaveContainerRecord} object in the Surreal wave_request table.
*
* @param token The request token associated with this request
* @param data A {@link WaveContainerRecord} object representing a Wave request record
*/
void saveContainerRequest(String token, WaveContainerRecord data)
void saveContainerRequest(WaveContainerRecord data)

/**
* Update a container request with the digest of the resolved request
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import groovy.transform.CompileStatic
import groovy.transform.ToString
import groovy.util.logging.Slf4j
import io.seqera.wave.api.ContainerConfig
import io.seqera.wave.api.FusionVersion
import io.seqera.wave.api.SubmitContainerTokenRequest
import io.seqera.wave.service.ContainerRequestData
import io.seqera.wave.tower.User
Expand All @@ -42,6 +41,12 @@ import static io.seqera.wave.util.DataTimeUtils.parseOffsetDateTime
@CompileStatic
class WaveContainerRecord {

/**
* wave request id, this will be the token
* This is container token and it is named as id for surrealdb requirement
*/
final String id

pditommaso marked this conversation as resolved.
Show resolved Hide resolved
/**
* The Tower user associated with the request
*/
Expand Down Expand Up @@ -158,7 +163,8 @@ class WaveContainerRecord {
*/
final String fusionVersion

WaveContainerRecord(SubmitContainerTokenRequest request, ContainerRequestData data, String waveImage, String addr, Instant expiration) {
WaveContainerRecord(SubmitContainerTokenRequest request, ContainerRequestData data, String token, String waveImage, String addr, Instant expiration) {
this.id = token
this.user = data.identity.user
this.workspaceId = request.towerWorkspaceId
this.containerImage = request.containerImage
Expand All @@ -184,6 +190,7 @@ class WaveContainerRecord {
}

WaveContainerRecord(WaveContainerRecord that, String sourceDigest, String waveDigest) {
this.id = that.id
this.user = that.user
this.workspaceId = that.workspaceId
this.containerImage = that.containerImage
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,8 @@ class LocalPersistenceService implements PersistenceService {
}

@Override
void saveContainerRequest(String token, WaveContainerRecord data) {
requestStore.put(token, data)
void saveContainerRequest(WaveContainerRecord data) {
requestStore.put(data.id, data)
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,15 +100,19 @@ class SurrealPersistenceService implements PersistenceService {

@Override
void saveBuild(WaveBuildRecord build) {
surrealDb.insertBuildAsync(getAuthorization(), build).subscribe({ result->
log.trace "Build request with id '$build.buildId' saved record: ${result}"
}, {error->
def msg = error.message
if( error instanceof HttpClientResponseException ){
msg += ":\n $error.response.body"
}
log.error("Error saving Build request record ${msg}\n${build}", error)
})
final query = "INSERT INTO wave_build ${JacksonHelper.toJson(build)}"
surrealDb
.sqlAsync(getAuthorization(), query)
.subscribe({result ->
log.trace "Conda file added in wave_build with buildId '$build.buildId': ${result}"
},
{error->
def msg = error.message
if( error instanceof HttpClientResponseException ){
msg += ":\n $error.response.body"
}
log.error("Error saving conda file in wave_build with buildId '$build.buildId => ${msg}\n", error)
})
}

@Override
Expand Down Expand Up @@ -167,16 +171,20 @@ class SurrealPersistenceService implements PersistenceService {
}

@Override
void saveContainerRequest(String token, WaveContainerRecord data) {
surrealDb.insertContainerRequestAsync(authorization, token, data).subscribe({ result->
log.trace "Container request with token '$token' saved record: ${result}"
}, {error->
def msg = error.message
if( error instanceof HttpClientResponseException ){
msg += ":\n $error.response.body"
}
log.error("Error saving container request record ${msg}\n${data}", error)
})
void saveContainerRequest(WaveContainerRecord data) {
final query = "INSERT INTO wave_request ${JacksonHelper.toJson(data)}"
surrealDb
.sqlAsync(getAuthorization(), query)
.subscribe({result ->
log.trace "Container request with token '$data.id' saved record: ${result}"
},
{error->
def msg = error.message
if( error instanceof HttpClientResponseException ){
msg += ":\n $error.response.body"
}
log.error("Error saving container request record ${msg}\n${data}", error)
})
}

void updateContainerRequest(String token, ContainerDigestPair digest) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,11 +194,12 @@ class ViewControllerTest extends Specification {

and:
def exp = Instant.now().plusSeconds(3600)
def container = new WaveContainerRecord(req, data, wave, addr, exp)
def token = '12345'
def container = new WaveContainerRecord(req, data, token, wave, addr, exp)


when:
persistenceService.saveContainerRequest(token, container)
persistenceService.saveContainerRequest(container)
and:
def request = HttpRequest.GET("/view/containers/${token}")
def response = client.toBlocking().exchange(request, String)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,10 @@ class WaveContainerRecordTest extends Specification {

when:
def exp = Instant.now().plusSeconds(3600)
def container = new WaveContainerRecord(req, data, wave, addr, exp)
def token = '1234'
def container = new WaveContainerRecord(req, data, token, wave, addr, exp)
then:
container.id == token
container.user == user
container.workspaceId == req.towerWorkspaceId
container.containerImage == req.containerImage
Expand Down Expand Up @@ -101,8 +103,10 @@ class WaveContainerRecordTest extends Specification {

when:
def exp = Instant.now().plusSeconds(3600)
def container = new WaveContainerRecord(req, data, wave, addr, exp)
def token = '1234'
def container = new WaveContainerRecord(req, data, token, wave, addr, exp)
then:
container.id == token
container.user == user
container.workspaceId == req.towerWorkspaceId
container.containerImage == req.containerImage
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -254,18 +254,38 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe
def addr = "100.200.300.400"
def exp = Instant.now().plusSeconds(3600)
and:
def request = new WaveContainerRecord(req, data, wave, addr, exp)
def request = new WaveContainerRecord(req, data, TOKEN, wave, addr, exp)

and:
persistence.saveContainerRequest(TOKEN, request)
persistence.saveContainerRequest(request)
and:
sleep 200 // <-- the above request is async, give time to save it

when:
def loaded = persistence.loadContainerRequest(TOKEN)
then:
loaded == request

then:
verifyAll(loaded) {
loaded.id == "wave_request:$request.id" //surrealdb will add table name in front of id
loaded.user == request.user
loaded.workspaceId == request.workspaceId
loaded.containerImage == request.containerImage
loaded.containerConfig == request.containerConfig
loaded.platform == request.platform
loaded.towerEndpoint == request.towerEndpoint
loaded.buildRepository == request.buildRepository
loaded.cacheRepository == request.cacheRepository
loaded.fingerprint == request.fingerprint
loaded.sourceImage == request.sourceImage
loaded.sourceDigest == request.sourceDigest
loaded.waveImage == request.waveImage
loaded.waveDigest == request.waveDigest
loaded.expiration == request.expiration
loaded.buildId == request.buildId
loaded.buildNew == request.buildNew
loaded.freeze == request.freeze
loaded.fusionVersion == request.fusionVersion
}

// should update the record
when:
Expand Down